2024-11-24 08:13:31,532 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-24 08:13:31,552 main DEBUG Took 0.017830 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 08:13:31,553 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 08:13:31,553 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 08:13:31,554 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 08:13:31,556 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:13:31,565 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 08:13:31,580 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:13:31,581 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:13:31,582 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:13:31,582 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:13:31,583 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:13:31,583 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:13:31,584 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:13:31,584 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:13:31,585 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:13:31,586 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:13:31,587 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:13:31,587 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:13:31,588 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:13:31,588 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:13:31,589 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:13:31,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:13:31,590 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:13:31,590 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:13:31,591 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:13:31,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:13:31,592 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:13:31,592 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:13:31,593 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:13:31,593 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 08:13:31,594 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:13:31,594 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 08:13:31,596 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 08:13:31,598 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 08:13:31,600 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 08:13:31,601 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 08:13:31,602 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 08:13:31,603 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 08:13:31,615 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 08:13:31,618 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 08:13:31,620 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 08:13:31,621 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 08:13:31,621 main DEBUG createAppenders(={Console}) 2024-11-24 08:13:31,622 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-24 08:13:31,622 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-24 08:13:31,623 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-24 08:13:31,623 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 08:13:31,624 main DEBUG OutputStream closed 2024-11-24 08:13:31,624 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 08:13:31,625 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 08:13:31,625 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-24 08:13:31,700 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 08:13:31,702 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 08:13:31,704 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 08:13:31,705 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 08:13:31,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 08:13:31,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 08:13:31,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 08:13:31,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 08:13:31,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 08:13:31,708 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 08:13:31,708 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 08:13:31,709 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 08:13:31,709 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 08:13:31,710 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 08:13:31,710 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 08:13:31,711 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 08:13:31,711 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 08:13:31,712 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 08:13:31,715 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 08:13:31,715 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-24 08:13:31,715 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 08:13:31,716 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-24T08:13:31,943 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369 2024-11-24 08:13:31,946 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 08:13:31,947 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T08:13:31,955 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-24T08:13:31,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T08:13:31,987 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/cluster_6ca8f465-b694-0817-15a9-f097eff6822b, deleteOnExit=true 2024-11-24T08:13:31,987 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T08:13:31,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/test.cache.data in system properties and HBase conf 2024-11-24T08:13:31,988 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T08:13:31,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/hadoop.log.dir in system properties and HBase conf 2024-11-24T08:13:31,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T08:13:31,990 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T08:13:31,990 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T08:13:32,076 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T08:13:32,168 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T08:13:32,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T08:13:32,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T08:13:32,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T08:13:32,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T08:13:32,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T08:13:32,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T08:13:32,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T08:13:32,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T08:13:32,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T08:13:32,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/nfs.dump.dir in system properties and HBase conf 2024-11-24T08:13:32,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/java.io.tmpdir in system properties and HBase conf 2024-11-24T08:13:32,176 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T08:13:32,176 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T08:13:32,177 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T08:13:32,979 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T08:13:33,059 INFO [Time-limited test {}] log.Log(170): Logging initialized @2194ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T08:13:33,138 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T08:13:33,204 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T08:13:33,224 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T08:13:33,224 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T08:13:33,226 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T08:13:33,240 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T08:13:33,243 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/hadoop.log.dir/,AVAILABLE} 2024-11-24T08:13:33,244 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T08:13:33,445 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/java.io.tmpdir/jetty-localhost-35129-hadoop-hdfs-3_4_1-tests_jar-_-any-11111900484002604956/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T08:13:33,458 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:35129} 2024-11-24T08:13:33,458 INFO [Time-limited test {}] server.Server(415): Started @2594ms 2024-11-24T08:13:33,905 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T08:13:33,912 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T08:13:33,913 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T08:13:33,913 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T08:13:33,914 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T08:13:33,914 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/hadoop.log.dir/,AVAILABLE} 2024-11-24T08:13:33,915 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T08:13:34,067 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/java.io.tmpdir/jetty-localhost-40633-hadoop-hdfs-3_4_1-tests_jar-_-any-16434994581549277923/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T08:13:34,068 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:40633} 2024-11-24T08:13:34,068 INFO [Time-limited test {}] server.Server(415): Started @3204ms 2024-11-24T08:13:34,131 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T08:13:34,597 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/cluster_6ca8f465-b694-0817-15a9-f097eff6822b/data/data1/current/BP-1872930679-172.17.0.2-1732436012737/current, will proceed with Du for space computation calculation, 2024-11-24T08:13:34,597 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/cluster_6ca8f465-b694-0817-15a9-f097eff6822b/data/data2/current/BP-1872930679-172.17.0.2-1732436012737/current, will proceed with Du for space computation calculation, 2024-11-24T08:13:34,671 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T08:13:34,724 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x634abaa51e7425b2 with lease ID 0x76d7330473e00c3e: Processing first storage report for DS-82eef8b7-ae38-469d-a445-802dbb4c6efc from datanode DatanodeRegistration(127.0.0.1:46185, datanodeUuid=67fa83ec-f3c9-41c1-adf0-2436a7ad0421, infoPort=33277, infoSecurePort=0, ipcPort=38431, storageInfo=lv=-57;cid=testClusterID;nsid=1134612293;c=1732436012737) 2024-11-24T08:13:34,727 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x634abaa51e7425b2 with lease ID 0x76d7330473e00c3e: from storage DS-82eef8b7-ae38-469d-a445-802dbb4c6efc node DatanodeRegistration(127.0.0.1:46185, datanodeUuid=67fa83ec-f3c9-41c1-adf0-2436a7ad0421, infoPort=33277, infoSecurePort=0, ipcPort=38431, storageInfo=lv=-57;cid=testClusterID;nsid=1134612293;c=1732436012737), blocks: 0, hasStaleStorage: true, processing time: 4 msecs, invalidatedBlocks: 0 2024-11-24T08:13:34,727 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x634abaa51e7425b2 with lease ID 0x76d7330473e00c3e: Processing first storage report for DS-f0e367e9-7277-49a8-ab80-c6b9f6fc74c1 from datanode DatanodeRegistration(127.0.0.1:46185, datanodeUuid=67fa83ec-f3c9-41c1-adf0-2436a7ad0421, infoPort=33277, infoSecurePort=0, ipcPort=38431, storageInfo=lv=-57;cid=testClusterID;nsid=1134612293;c=1732436012737) 2024-11-24T08:13:34,728 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x634abaa51e7425b2 with lease ID 0x76d7330473e00c3e: from storage DS-f0e367e9-7277-49a8-ab80-c6b9f6fc74c1 node DatanodeRegistration(127.0.0.1:46185, datanodeUuid=67fa83ec-f3c9-41c1-adf0-2436a7ad0421, infoPort=33277, infoSecurePort=0, ipcPort=38431, storageInfo=lv=-57;cid=testClusterID;nsid=1134612293;c=1732436012737), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T08:13:34,801 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369 2024-11-24T08:13:34,880 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/cluster_6ca8f465-b694-0817-15a9-f097eff6822b/zookeeper_0, clientPort=63500, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/cluster_6ca8f465-b694-0817-15a9-f097eff6822b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/cluster_6ca8f465-b694-0817-15a9-f097eff6822b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T08:13:34,889 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63500 2024-11-24T08:13:34,903 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:13:34,908 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:13:35,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741825_1001 (size=7) 2024-11-24T08:13:35,568 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6 with version=8 2024-11-24T08:13:35,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/hbase-staging 2024-11-24T08:13:35,696 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T08:13:36,004 INFO [Time-limited test {}] client.ConnectionUtils(128): master/30c28c82771d:0 server-side Connection retries=45 2024-11-24T08:13:36,020 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:13:36,021 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T08:13:36,028 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T08:13:36,028 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:13:36,028 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T08:13:36,218 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T08:13:36,305 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T08:13:36,315 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T08:13:36,320 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T08:13:36,354 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3062 (auto-detected) 2024-11-24T08:13:36,355 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T08:13:36,384 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38469 2024-11-24T08:13:36,410 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38469 connecting to ZooKeeper ensemble=127.0.0.1:63500 2024-11-24T08:13:36,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:384690x0, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T08:13:36,461 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38469-0x1014909a19b0000 connected 2024-11-24T08:13:36,527 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:13:36,530 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:13:36,544 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:13:36,550 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6, hbase.cluster.distributed=false 2024-11-24T08:13:36,584 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T08:13:36,609 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38469 2024-11-24T08:13:36,612 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38469 2024-11-24T08:13:36,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38469 2024-11-24T08:13:36,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38469 2024-11-24T08:13:36,632 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38469 2024-11-24T08:13:36,779 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/30c28c82771d:0 server-side Connection retries=45 2024-11-24T08:13:36,781 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:13:36,783 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T08:13:36,785 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T08:13:36,785 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T08:13:36,785 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T08:13:36,789 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T08:13:36,795 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T08:13:36,809 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45087 2024-11-24T08:13:36,811 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45087 connecting to ZooKeeper ensemble=127.0.0.1:63500 2024-11-24T08:13:36,813 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:13:36,819 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:13:36,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:450870x0, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T08:13:36,851 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:450870x0, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:13:36,853 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45087-0x1014909a19b0001 connected 2024-11-24T08:13:36,857 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T08:13:36,874 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T08:13:36,877 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T08:13:36,884 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T08:13:36,887 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45087 2024-11-24T08:13:36,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45087 2024-11-24T08:13:36,892 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45087 2024-11-24T08:13:36,896 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45087 2024-11-24T08:13:36,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45087 2024-11-24T08:13:36,916 DEBUG [M:0;30c28c82771d:38469 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;30c28c82771d:38469 2024-11-24T08:13:36,917 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/30c28c82771d,38469,1732436015766 2024-11-24T08:13:36,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:13:36,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:13:36,929 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/30c28c82771d,38469,1732436015766 2024-11-24T08:13:36,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:13:36,955 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T08:13:36,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T08:13:36,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:13:36,957 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/30c28c82771d,38469,1732436015766 from backup master directory 2024-11-24T08:13:36,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:13:36,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/30c28c82771d,38469,1732436015766 2024-11-24T08:13:36,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T08:13:36,963 WARN [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T08:13:36,963 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=30c28c82771d,38469,1732436015766 2024-11-24T08:13:36,966 INFO [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T08:13:36,968 INFO [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T08:13:37,051 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/hbase.id] with ID: d039f08b-cffb-43d0-831a-6a176a1b421f 2024-11-24T08:13:37,052 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/.tmp/hbase.id 2024-11-24T08:13:37,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741826_1002 (size=42) 2024-11-24T08:13:37,073 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/.tmp/hbase.id]:[hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/hbase.id] 2024-11-24T08:13:37,135 INFO [master/30c28c82771d:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T08:13:37,141 INFO [master/30c28c82771d:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T08:13:37,162 INFO [master/30c28c82771d:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-24T08:13:37,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:13:37,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:13:37,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741827_1003 (size=196) 2024-11-24T08:13:37,201 INFO [master/30c28c82771d:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:13:37,204 INFO [master/30c28c82771d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T08:13:37,220 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:37,225 INFO [master/30c28c82771d:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:13:37,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741828_1004 (size=1189) 2024-11-24T08:13:37,675 INFO [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store 2024-11-24T08:13:37,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741829_1005 (size=34) 2024-11-24T08:13:37,703 INFO [master/30c28c82771d:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T08:13:37,707 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:13:37,709 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T08:13:37,709 INFO [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:13:37,709 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:13:37,711 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T08:13:37,711 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:13:37,711 INFO [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:13:37,712 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732436017709Disabling compacts and flushes for region at 1732436017709Disabling writes for close at 1732436017711 (+2 ms)Writing region close event to WAL at 1732436017711Closed at 1732436017711 2024-11-24T08:13:37,715 WARN [master/30c28c82771d:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/.initializing 2024-11-24T08:13:37,715 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/WALs/30c28c82771d,38469,1732436015766 2024-11-24T08:13:37,725 INFO [master/30c28c82771d:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:13:37,744 INFO [master/30c28c82771d:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=30c28c82771d%2C38469%2C1732436015766, suffix=, logDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/WALs/30c28c82771d,38469,1732436015766, archiveDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/oldWALs, maxLogs=10 2024-11-24T08:13:37,780 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/WALs/30c28c82771d,38469,1732436015766/30c28c82771d%2C38469%2C1732436015766.1732436017750, exclude list is [], retry=0 2024-11-24T08:13:37,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46185,DS-82eef8b7-ae38-469d-a445-802dbb4c6efc,DISK] 2024-11-24T08:13:37,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T08:13:37,849 INFO [master/30c28c82771d:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/WALs/30c28c82771d,38469,1732436015766/30c28c82771d%2C38469%2C1732436015766.1732436017750 2024-11-24T08:13:37,850 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33277:33277)] 2024-11-24T08:13:37,850 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:13:37,851 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:13:37,854 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:13:37,855 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:13:37,906 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:13:37,942 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T08:13:37,947 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:37,950 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:13:37,951 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:13:37,954 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T08:13:37,955 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:37,956 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:13:37,956 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:13:37,959 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T08:13:37,959 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:37,960 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:13:37,960 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:13:37,963 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T08:13:37,963 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:37,964 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:13:37,965 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:13:37,969 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:13:37,970 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:13:37,985 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:13:37,986 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:13:37,990 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T08:13:37,994 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T08:13:37,999 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:13:38,000 INFO [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59016114, jitterRate=-0.12059137225151062}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T08:13:38,006 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732436017870Initializing all the Stores at 1732436017873 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732436017873Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436017874 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436017875 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436017875Cleaning up temporary data from old regions at 1732436017986 (+111 ms)Region opened successfully at 1732436018006 (+20 ms) 2024-11-24T08:13:38,007 INFO [master/30c28c82771d:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T08:13:38,040 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ea1de71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=30c28c82771d/172.17.0.2:0 2024-11-24T08:13:38,072 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T08:13:38,084 INFO [master/30c28c82771d:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T08:13:38,084 INFO [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T08:13:38,087 INFO [master/30c28c82771d:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T08:13:38,089 INFO [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T08:13:38,093 INFO [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T08:13:38,094 INFO [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T08:13:38,120 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T08:13:38,131 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T08:13:38,133 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T08:13:38,135 INFO [master/30c28c82771d:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T08:13:38,137 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T08:13:38,138 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T08:13:38,140 INFO [master/30c28c82771d:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T08:13:38,144 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T08:13:38,145 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T08:13:38,147 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T08:13:38,150 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T08:13:38,168 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T08:13:38,169 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T08:13:38,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T08:13:38,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T08:13:38,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:13:38,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:13:38,176 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=30c28c82771d,38469,1732436015766, sessionid=0x1014909a19b0000, setting cluster-up flag (Was=false) 2024-11-24T08:13:38,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:13:38,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:13:38,198 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T08:13:38,200 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=30c28c82771d,38469,1732436015766 2024-11-24T08:13:38,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:13:38,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:13:38,213 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T08:13:38,214 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=30c28c82771d,38469,1732436015766 2024-11-24T08:13:38,222 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T08:13:38,303 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T08:13:38,304 INFO [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(746): ClusterId : d039f08b-cffb-43d0-831a-6a176a1b421f 2024-11-24T08:13:38,307 DEBUG [RS:0;30c28c82771d:45087 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T08:13:38,312 DEBUG [RS:0;30c28c82771d:45087 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T08:13:38,312 DEBUG [RS:0;30c28c82771d:45087 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T08:13:38,314 INFO [master/30c28c82771d:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T08:13:38,315 DEBUG [RS:0;30c28c82771d:45087 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T08:13:38,315 DEBUG [RS:0;30c28c82771d:45087 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dba2e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=30c28c82771d/172.17.0.2:0 2024-11-24T08:13:38,320 INFO [master/30c28c82771d:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T08:13:38,326 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 30c28c82771d,38469,1732436015766 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T08:13:38,332 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/30c28c82771d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:13:38,333 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;30c28c82771d:45087 2024-11-24T08:13:38,333 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/30c28c82771d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:13:38,333 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/30c28c82771d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:13:38,333 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/30c28c82771d:0, corePoolSize=5, maxPoolSize=5 2024-11-24T08:13:38,333 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/30c28c82771d:0, corePoolSize=10, maxPoolSize=10 2024-11-24T08:13:38,333 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:13:38,334 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/30c28c82771d:0, corePoolSize=2, maxPoolSize=2 2024-11-24T08:13:38,334 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:13:38,336 INFO [RS:0;30c28c82771d:45087 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T08:13:38,336 INFO [RS:0;30c28c82771d:45087 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T08:13:38,337 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T08:13:38,337 INFO [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732436048337 2024-11-24T08:13:38,339 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T08:13:38,339 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T08:13:38,340 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T08:13:38,340 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T08:13:38,340 INFO [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(2659): reportForDuty to master=30c28c82771d,38469,1732436015766 with port=45087, startcode=1732436016722 2024-11-24T08:13:38,344 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T08:13:38,344 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T08:13:38,344 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T08:13:38,345 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T08:13:38,345 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:38,348 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:38,348 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T08:13:38,349 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T08:13:38,350 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T08:13:38,350 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T08:13:38,352 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T08:13:38,353 INFO [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T08:13:38,355 DEBUG [RS:0;30c28c82771d:45087 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T08:13:38,356 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/30c28c82771d:0:becomeActiveMaster-HFileCleaner.large.0-1732436018354,5,FailOnTimeoutGroup] 2024-11-24T08:13:38,359 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/30c28c82771d:0:becomeActiveMaster-HFileCleaner.small.0-1732436018356,5,FailOnTimeoutGroup] 2024-11-24T08:13:38,360 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:38,360 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T08:13:38,361 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:38,362 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:38,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741831_1007 (size=1321) 2024-11-24T08:13:38,364 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T08:13:38,365 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6 2024-11-24T08:13:38,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741832_1008 (size=32) 2024-11-24T08:13:38,390 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58965, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T08:13:38,396 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38469 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 30c28c82771d,45087,1732436016722 2024-11-24T08:13:38,399 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38469 {}] master.ServerManager(517): Registering regionserver=30c28c82771d,45087,1732436016722 2024-11-24T08:13:38,415 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6 2024-11-24T08:13:38,416 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44153 2024-11-24T08:13:38,416 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T08:13:38,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:13:38,421 DEBUG [RS:0;30c28c82771d:45087 {}] zookeeper.ZKUtil(111): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/30c28c82771d,45087,1732436016722 2024-11-24T08:13:38,422 WARN [RS:0;30c28c82771d:45087 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T08:13:38,422 INFO [RS:0;30c28c82771d:45087 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:13:38,422 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/WALs/30c28c82771d,45087,1732436016722 2024-11-24T08:13:38,424 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [30c28c82771d,45087,1732436016722] 2024-11-24T08:13:38,456 INFO [RS:0;30c28c82771d:45087 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T08:13:38,471 INFO [RS:0;30c28c82771d:45087 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T08:13:38,477 INFO [RS:0;30c28c82771d:45087 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T08:13:38,477 INFO [RS:0;30c28c82771d:45087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:38,478 INFO [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T08:13:38,484 INFO [RS:0;30c28c82771d:45087 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T08:13:38,485 INFO [RS:0;30c28c82771d:45087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:38,485 DEBUG [RS:0;30c28c82771d:45087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:13:38,485 DEBUG [RS:0;30c28c82771d:45087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:13:38,485 DEBUG [RS:0;30c28c82771d:45087 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:13:38,486 DEBUG [RS:0;30c28c82771d:45087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:13:38,486 DEBUG [RS:0;30c28c82771d:45087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:13:38,486 DEBUG [RS:0;30c28c82771d:45087 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/30c28c82771d:0, corePoolSize=2, maxPoolSize=2 2024-11-24T08:13:38,486 DEBUG [RS:0;30c28c82771d:45087 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:13:38,486 DEBUG [RS:0;30c28c82771d:45087 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:13:38,487 DEBUG [RS:0;30c28c82771d:45087 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:13:38,487 DEBUG [RS:0;30c28c82771d:45087 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:13:38,487 DEBUG [RS:0;30c28c82771d:45087 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:13:38,487 DEBUG [RS:0;30c28c82771d:45087 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/30c28c82771d:0, corePoolSize=1, maxPoolSize=1 2024-11-24T08:13:38,487 DEBUG [RS:0;30c28c82771d:45087 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/30c28c82771d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:13:38,487 DEBUG [RS:0;30c28c82771d:45087 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0, corePoolSize=3, maxPoolSize=3 2024-11-24T08:13:38,488 INFO [RS:0;30c28c82771d:45087 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:38,488 INFO [RS:0;30c28c82771d:45087 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:38,489 INFO [RS:0;30c28c82771d:45087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:38,489 INFO [RS:0;30c28c82771d:45087 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:38,489 INFO [RS:0;30c28c82771d:45087 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:38,489 INFO [RS:0;30c28c82771d:45087 {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,45087,1732436016722-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T08:13:38,509 INFO [RS:0;30c28c82771d:45087 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T08:13:38,511 INFO [RS:0;30c28c82771d:45087 {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,45087,1732436016722-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:38,511 INFO [RS:0;30c28c82771d:45087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:38,511 INFO [RS:0;30c28c82771d:45087 {}] regionserver.Replication(171): 30c28c82771d,45087,1732436016722 started 2024-11-24T08:13:38,530 INFO [RS:0;30c28c82771d:45087 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:38,530 INFO [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1482): Serving as 30c28c82771d,45087,1732436016722, RpcServer on 30c28c82771d/172.17.0.2:45087, sessionid=0x1014909a19b0001 2024-11-24T08:13:38,531 DEBUG [RS:0;30c28c82771d:45087 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T08:13:38,531 DEBUG [RS:0;30c28c82771d:45087 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 30c28c82771d,45087,1732436016722 2024-11-24T08:13:38,531 DEBUG [RS:0;30c28c82771d:45087 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '30c28c82771d,45087,1732436016722' 2024-11-24T08:13:38,531 DEBUG [RS:0;30c28c82771d:45087 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T08:13:38,532 DEBUG [RS:0;30c28c82771d:45087 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T08:13:38,533 DEBUG [RS:0;30c28c82771d:45087 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T08:13:38,533 DEBUG [RS:0;30c28c82771d:45087 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T08:13:38,533 DEBUG [RS:0;30c28c82771d:45087 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 30c28c82771d,45087,1732436016722 2024-11-24T08:13:38,533 DEBUG [RS:0;30c28c82771d:45087 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '30c28c82771d,45087,1732436016722' 2024-11-24T08:13:38,533 DEBUG [RS:0;30c28c82771d:45087 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T08:13:38,534 DEBUG [RS:0;30c28c82771d:45087 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T08:13:38,534 DEBUG [RS:0;30c28c82771d:45087 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T08:13:38,535 INFO [RS:0;30c28c82771d:45087 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T08:13:38,535 INFO [RS:0;30c28c82771d:45087 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T08:13:38,640 INFO [RS:0;30c28c82771d:45087 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T08:13:38,643 INFO [RS:0;30c28c82771d:45087 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=30c28c82771d%2C45087%2C1732436016722, suffix=, logDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/WALs/30c28c82771d,45087,1732436016722, archiveDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/oldWALs, maxLogs=32 2024-11-24T08:13:38,660 DEBUG [RS:0;30c28c82771d:45087 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/WALs/30c28c82771d,45087,1732436016722/30c28c82771d%2C45087%2C1732436016722.1732436018645, exclude list is [], retry=0 2024-11-24T08:13:38,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46185,DS-82eef8b7-ae38-469d-a445-802dbb4c6efc,DISK] 2024-11-24T08:13:38,669 INFO [RS:0;30c28c82771d:45087 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/WALs/30c28c82771d,45087,1732436016722/30c28c82771d%2C45087%2C1732436016722.1732436018645 2024-11-24T08:13:38,669 DEBUG [RS:0;30c28c82771d:45087 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33277:33277)] 2024-11-24T08:13:38,784 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:13:38,787 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T08:13:38,789 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T08:13:38,790 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:38,790 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:13:38,791 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T08:13:38,793 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T08:13:38,793 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:38,794 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:13:38,794 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T08:13:38,797 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T08:13:38,797 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:38,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:13:38,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T08:13:38,800 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T08:13:38,801 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:38,801 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:13:38,802 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T08:13:38,803 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740 2024-11-24T08:13:38,804 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740 2024-11-24T08:13:38,807 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T08:13:38,807 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T08:13:38,808 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T08:13:38,811 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T08:13:38,815 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:13:38,816 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71056454, jitterRate=0.05882367491722107}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T08:13:38,818 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732436018784Initializing all the Stores at 1732436018786 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732436018786Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732436018786Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436018787 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732436018787Cleaning up temporary data from old regions at 1732436018807 (+20 ms)Region opened successfully at 1732436018818 (+11 ms) 2024-11-24T08:13:38,818 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T08:13:38,818 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T08:13:38,819 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T08:13:38,819 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T08:13:38,819 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T08:13:38,820 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T08:13:38,821 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732436018818Disabling compacts and flushes for region at 1732436018818Disabling writes for close at 1732436018819 (+1 ms)Writing region close event to WAL at 1732436018820 (+1 ms)Closed at 1732436018820 2024-11-24T08:13:38,825 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T08:13:38,825 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T08:13:38,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T08:13:38,840 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T08:13:38,842 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T08:13:38,995 DEBUG [30c28c82771d:38469 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T08:13:39,008 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:13:39,015 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 30c28c82771d,45087,1732436016722, state=OPENING 2024-11-24T08:13:39,023 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T08:13:39,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:13:39,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:13:39,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:13:39,026 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:13:39,027 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T08:13:39,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:13:39,206 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T08:13:39,209 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60921, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T08:13:39,221 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T08:13:39,221 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T08:13:39,221 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T08:13:39,224 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=30c28c82771d%2C45087%2C1732436016722.meta, suffix=.meta, logDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/WALs/30c28c82771d,45087,1732436016722, archiveDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/oldWALs, maxLogs=32 2024-11-24T08:13:39,241 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/WALs/30c28c82771d,45087,1732436016722/30c28c82771d%2C45087%2C1732436016722.meta.1732436019226.meta, exclude list is [], retry=0 2024-11-24T08:13:39,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46185,DS-82eef8b7-ae38-469d-a445-802dbb4c6efc,DISK] 2024-11-24T08:13:39,247 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/WALs/30c28c82771d,45087,1732436016722/30c28c82771d%2C45087%2C1732436016722.meta.1732436019226.meta 2024-11-24T08:13:39,247 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33277:33277)] 2024-11-24T08:13:39,248 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:13:39,249 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T08:13:39,252 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T08:13:39,256 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T08:13:39,261 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T08:13:39,261 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:13:39,261 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T08:13:39,262 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T08:13:39,264 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T08:13:39,266 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T08:13:39,266 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:39,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:13:39,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T08:13:39,269 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T08:13:39,269 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:39,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:13:39,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T08:13:39,272 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T08:13:39,272 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:39,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:13:39,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T08:13:39,277 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T08:13:39,277 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:39,278 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T08:13:39,278 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T08:13:39,280 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740 2024-11-24T08:13:39,283 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740 2024-11-24T08:13:39,285 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T08:13:39,285 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T08:13:39,286 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T08:13:39,289 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T08:13:39,291 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65809359, jitterRate=-0.019364133477211}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T08:13:39,291 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T08:13:39,293 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732436019262Writing region info on filesystem at 1732436019262Initializing all the Stores at 1732436019264 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732436019264Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732436019264Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436019264Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732436019264Cleaning up temporary data from old regions at 1732436019286 (+22 ms)Running coprocessor post-open hooks at 1732436019291 (+5 ms)Region opened successfully at 1732436019293 (+2 ms) 2024-11-24T08:13:39,301 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732436019197 2024-11-24T08:13:39,313 DEBUG [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T08:13:39,314 INFO [RS_OPEN_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T08:13:39,315 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:13:39,317 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 30c28c82771d,45087,1732436016722, state=OPEN 2024-11-24T08:13:39,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T08:13:39,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T08:13:39,327 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:13:39,327 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T08:13:39,327 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:39,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T08:13:39,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=30c28c82771d,45087,1732436016722 in 297 msec 2024-11-24T08:13:39,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T08:13:39,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 503 msec 2024-11-24T08:13:39,340 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T08:13:39,340 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T08:13:39,360 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:13:39,362 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:13:39,382 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:13:39,385 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44501, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:13:39,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1490 sec 2024-11-24T08:13:39,403 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732436019403, completionTime=-1 2024-11-24T08:13:39,406 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-24T08:13:39,406 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T08:13:39,436 INFO [master/30c28c82771d:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-24T08:13:39,436 INFO [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732436079436 2024-11-24T08:13:39,437 INFO [master/30c28c82771d:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732436139436 2024-11-24T08:13:39,437 INFO [master/30c28c82771d:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-11-24T08:13:39,439 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,38469,1732436015766-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:39,440 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,38469,1732436015766-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:39,440 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,38469,1732436015766-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:39,441 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-30c28c82771d:38469, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:39,442 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:39,444 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:39,449 DEBUG [master/30c28c82771d:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T08:13:39,470 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.507sec 2024-11-24T08:13:39,471 INFO [master/30c28c82771d:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T08:13:39,472 INFO [master/30c28c82771d:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T08:13:39,473 INFO [master/30c28c82771d:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T08:13:39,474 INFO [master/30c28c82771d:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T08:13:39,474 INFO [master/30c28c82771d:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T08:13:39,475 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,38469,1732436015766-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T08:13:39,475 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,38469,1732436015766-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T08:13:39,483 DEBUG [master/30c28c82771d:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T08:13:39,484 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T08:13:39,485 INFO [master/30c28c82771d:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=30c28c82771d,38469,1732436015766-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T08:13:39,514 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:39,516 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T08:13:39,516 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T08:13:39,520 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:13:39,522 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:13:39,531 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:13:39,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:13:39,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:13:39,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f93b2f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:39,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:13:39,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:13:39,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:13:39,543 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:13:39,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:39,546 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:13:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:13:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:13:39,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:13:39,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=30c28c82771d,38469,1732436015766 2024-11-24T08:13:39,589 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=238, ProcessCount=11, AvailableMemoryMB=7314 2024-11-24T08:13:39,598 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T08:13:39,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 30c28c82771d,38469,1732436015766 2024-11-24T08:13:39,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a16ee17 2024-11-24T08:13:39,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T08:13:39,610 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T08:13:39,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T08:13:39,618 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:13:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T08:13:39,630 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T08:13:39,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-24T08:13:39,633 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:39,635 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T08:13:39,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:13:39,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741835_1011 (size=1061) 2024-11-24T08:13:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:13:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:13:40,072 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6 2024-11-24T08:13:40,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741836_1012 (size=53) 2024-11-24T08:13:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:13:40,482 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:13:40,482 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 187d8f78b18a17144230e52cfd716831, disabling compactions & flushes 2024-11-24T08:13:40,482 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:40,482 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:40,483 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. after waiting 0 ms 2024-11-24T08:13:40,483 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:40,483 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:40,483 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 187d8f78b18a17144230e52cfd716831: Waiting for close lock at 1732436020482Disabling compacts and flushes for region at 1732436020482Disabling writes for close at 1732436020483 (+1 ms)Writing region close event to WAL at 1732436020483Closed at 1732436020483 2024-11-24T08:13:40,485 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T08:13:40,491 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732436020486"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732436020486"}]},"ts":"1732436020486"} 2024-11-24T08:13:40,496 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T08:13:40,499 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T08:13:40,502 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436020499"}]},"ts":"1732436020499"} 2024-11-24T08:13:40,507 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T08:13:40,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=187d8f78b18a17144230e52cfd716831, ASSIGN}] 2024-11-24T08:13:40,512 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=187d8f78b18a17144230e52cfd716831, ASSIGN 2024-11-24T08:13:40,514 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=187d8f78b18a17144230e52cfd716831, ASSIGN; state=OFFLINE, location=30c28c82771d,45087,1732436016722; forceNewPlan=false, retain=false 2024-11-24T08:13:40,666 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=187d8f78b18a17144230e52cfd716831, regionState=OPENING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:13:40,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=187d8f78b18a17144230e52cfd716831, ASSIGN because future has completed 2024-11-24T08:13:40,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:13:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:13:40,849 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:40,849 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:13:40,850 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:40,850 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:13:40,850 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:40,850 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:40,853 INFO [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:40,858 INFO [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:13:40,858 INFO [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 187d8f78b18a17144230e52cfd716831 columnFamilyName A 2024-11-24T08:13:40,858 DEBUG [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:40,860 INFO [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] regionserver.HStore(327): Store=187d8f78b18a17144230e52cfd716831/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:13:40,860 INFO [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:40,865 INFO [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:13:40,866 INFO [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 187d8f78b18a17144230e52cfd716831 columnFamilyName B 2024-11-24T08:13:40,866 DEBUG [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:40,867 INFO [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] regionserver.HStore(327): Store=187d8f78b18a17144230e52cfd716831/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:13:40,867 INFO [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:40,870 INFO [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:13:40,870 INFO [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 187d8f78b18a17144230e52cfd716831 columnFamilyName C 2024-11-24T08:13:40,870 DEBUG [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:13:40,871 INFO [StoreOpener-187d8f78b18a17144230e52cfd716831-1 {}] regionserver.HStore(327): Store=187d8f78b18a17144230e52cfd716831/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:13:40,872 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:40,872 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:40,874 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:40,875 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:40,884 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:40,885 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:40,886 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T08:13:40,890 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:40,895 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:13:40,897 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 187d8f78b18a17144230e52cfd716831; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69089176, jitterRate=0.02950894832611084}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T08:13:40,897 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:40,900 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 187d8f78b18a17144230e52cfd716831: Running coprocessor pre-open hook at 1732436020850Writing region info on filesystem at 1732436020850Initializing all the Stores at 1732436020853 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436020853Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436020853Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436020853Cleaning up temporary data from old regions at 1732436020885 (+32 ms)Running coprocessor post-open hooks at 1732436020897 (+12 ms)Region opened successfully at 1732436020900 (+3 ms) 2024-11-24T08:13:40,903 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., pid=6, masterSystemTime=1732436020830 2024-11-24T08:13:40,909 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:40,909 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:40,911 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=187d8f78b18a17144230e52cfd716831, regionState=OPEN, openSeqNum=2, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:13:40,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:13:40,925 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T08:13:40,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 in 244 msec 2024-11-24T08:13:40,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T08:13:40,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=187d8f78b18a17144230e52cfd716831, ASSIGN in 416 msec 2024-11-24T08:13:40,932 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T08:13:40,932 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436020932"}]},"ts":"1732436020932"} 2024-11-24T08:13:40,936 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T08:13:40,938 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T08:13:40,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3170 sec 2024-11-24T08:13:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T08:13:41,812 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:13:41,819 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,819 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:13:41,819 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:13:41,821 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:13:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:13:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:13:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@700f0420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:13:41,823 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:13:41,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:13:41,826 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:13:41,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c3786e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,828 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:13:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:13:41,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:13:41,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:13:41,835 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T08:13:41,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 30c28c82771d,38469,1732436015766 2024-11-24T08:13:41,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c2fcd6 2024-11-24T08:13:41,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T08:13:41,841 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T08:13:41,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,847 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:13:41,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:13:41,849 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:13:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:13:41,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:13:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d22a118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:13:41,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:13:41,856 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:13:41,858 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:13:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,865 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,865 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:13:41,865 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:13:41,867 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:13:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:13:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:13:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6ed1ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:13:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:13:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:13:41,885 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:13:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:13:41,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:13:41,889 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:13:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:13:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:13:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef39c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:13:41,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:13:41,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:13:41,894 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:13:41,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7f3cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:13:41,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:13:41,913 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:13:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:13:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:13:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bd026af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:13:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:13:41,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:13:41,927 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:13:41,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@583cb1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,932 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,932 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:13:41,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:13:41,934 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:13:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:13:41,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:13:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e3ab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:13:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:13:41,935 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:13:41,936 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:13:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c4ddcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,939 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:13:41,939 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:13:41,940 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:13:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:13:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:13:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38cde2e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:13:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:13:41,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:13:41,943 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:13:41,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@332acafc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,953 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:13:41,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:13:41,954 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:13:41,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:13:41,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:13:41,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2db8e414, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,955 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:13:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:13:41,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:13:41,957 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:13:41,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d230658, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:13:41,960 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:13:41,961 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:13:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:13:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:13:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54cc6db2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,961 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:13:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:13:41,962 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:13:41,963 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:13:41,964 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af61fa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:13:41,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:13:41,970 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:13:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:13:41,970 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:13:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d5cf1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:13:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:13:41,971 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:13:41,973 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:13:41,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@caecd72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:13:41,981 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:13:41,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:13:41,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:13:41,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:13:41,986 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:13:41,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:13:41,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:13:41,990 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:13:41,992 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:13:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:13:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:13:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:13:41,995 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:13:41,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46212, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:13:42,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:13:42,006 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:13:42,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:13:42,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:13:42,011 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:13:42,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:13:42,021 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:13:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:13:42,024 DEBUG [Thread-160 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:13:42,025 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:13:42,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:13:42,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:13:42,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:13:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:13:42,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:13:42,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:13:42,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:13:42,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-24T08:13:42,057 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:13:42,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T08:13:42,059 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:13:42,061 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:13:42,065 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:13:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:13:42,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:13:42,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:13:42,070 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:13:42,072 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:13:42,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:13:42,074 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:13:42,075 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:13:42,078 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46262, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:13:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:13:42,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:13:42,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:13:42,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:42,084 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:13:42,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:13:42,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:13:42,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:42,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:42,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:42,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:42,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:42,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T08:13:42,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T08:13:42,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:42,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:42,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:42,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/523bef2d65a24f40a97c00547145caad is 50, key is test_row_0/A:col10/1732436022064/Put/seqid=0 2024-11-24T08:13:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436082223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436082219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732436022182, max timestamp=1732436022232Num uniques 859; 2024-11-24T08:13:42,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-24T08:13:42,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732436022182, max timestamp=1732436022232Num uniques 859; 2024-11-24T08:13:42,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-24T08:13:42,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1732436022182, max timestamp=1732436022232Num uniques 859; 2024-11-24T08:13:42,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-24T08:13:42,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741837_1013 (size=12001) 2024-11-24T08:13:42,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/523bef2d65a24f40a97c00547145caad 2024-11-24T08:13:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T08:13:42,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436082370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436082375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436082379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T08:13:42,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/807a070f6b2b47759cde355f70382322 is 50, key is test_row_0/B:col10/1732436022064/Put/seqid=0 2024-11-24T08:13:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:42,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436082430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436082431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741838_1014 (size=12001) 2024-11-24T08:13:42,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436082501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436082511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436082512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T08:13:42,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:42,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:42,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:42,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436082652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436082652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T08:13:42,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436082721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436082726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436082726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T08:13:42,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:42,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:42,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:42,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/807a070f6b2b47759cde355f70382322 2024-11-24T08:13:42,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T08:13:42,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:42,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:42,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:42,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:42,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/0762a25cddd54f39a2c59058e5733b04 is 50, key is test_row_0/C:col10/1732436022064/Put/seqid=0 2024-11-24T08:13:42,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741839_1015 (size=12001) 2024-11-24T08:13:42,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/0762a25cddd54f39a2c59058e5733b04 2024-11-24T08:13:42,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/523bef2d65a24f40a97c00547145caad as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/523bef2d65a24f40a97c00547145caad 2024-11-24T08:13:42,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/523bef2d65a24f40a97c00547145caad, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T08:13:42,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436082971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:42,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436082971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:42,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:42,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/807a070f6b2b47759cde355f70382322 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/807a070f6b2b47759cde355f70382322 2024-11-24T08:13:42,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/807a070f6b2b47759cde355f70382322, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T08:13:43,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/0762a25cddd54f39a2c59058e5733b04 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/0762a25cddd54f39a2c59058e5733b04 2024-11-24T08:13:43,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/0762a25cddd54f39a2c59058e5733b04, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T08:13:43,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 187d8f78b18a17144230e52cfd716831 in 932ms, sequenceid=12, compaction requested=false 2024-11-24T08:13:43,023 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T08:13:43,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:43,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-24T08:13:43,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:43,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:13:43,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:43,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:13:43,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:43,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:13:43,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T08:13:43,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:43,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:43,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:43,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/c9ac77af26514e0a845fdb38abd37deb is 50, key is test_row_0/A:col10/1732436023046/Put/seqid=0 2024-11-24T08:13:43,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741840_1016 (size=14341) 2024-11-24T08:13:43,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436083128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:43,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:43,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:43,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436083129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:43,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:43,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:43,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436083135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:43,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:43,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T08:13:43,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T08:13:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:43,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:43,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:43,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436083261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:43,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:43,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436083261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:43,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436083262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:43,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,270 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:43,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T08:13:43,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:43,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:43,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:43,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436083482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436083482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:43,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:43,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436083482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:43,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:43,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436083483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:43,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/c9ac77af26514e0a845fdb38abd37deb 2024-11-24T08:13:43,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:43,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:43,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436083495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:43,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:43,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/e75ad5ead35d47d9ab581af4726ead02 is 50, key is test_row_0/B:col10/1732436023046/Put/seqid=0 2024-11-24T08:13:43,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T08:13:43,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:43,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:43,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:43,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741841_1017 (size=12001) 2024-11-24T08:13:43,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/e75ad5ead35d47d9ab581af4726ead02 2024-11-24T08:13:43,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/120c75f2771d4716a95baa0e3d448af2 is 50, key is test_row_0/C:col10/1732436023046/Put/seqid=0 2024-11-24T08:13:43,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741842_1018 (size=12001) 2024-11-24T08:13:43,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T08:13:43,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:43,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:43,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:43,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:43,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436083801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:43,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436083801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:43,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436083801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:43,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:43,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T08:13:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:43,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:43,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:44,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T08:13:44,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:44,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:44,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:44,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:44,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:44,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/120c75f2771d4716a95baa0e3d448af2 2024-11-24T08:13:44,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/c9ac77af26514e0a845fdb38abd37deb as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/c9ac77af26514e0a845fdb38abd37deb 2024-11-24T08:13:44,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/c9ac77af26514e0a845fdb38abd37deb, entries=200, sequenceid=46, filesize=14.0 K 2024-11-24T08:13:44,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/e75ad5ead35d47d9ab581af4726ead02 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e75ad5ead35d47d9ab581af4726ead02 2024-11-24T08:13:44,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e75ad5ead35d47d9ab581af4726ead02, entries=150, sequenceid=46, filesize=11.7 K 2024-11-24T08:13:44,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/120c75f2771d4716a95baa0e3d448af2 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/120c75f2771d4716a95baa0e3d448af2 2024-11-24T08:13:44,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/120c75f2771d4716a95baa0e3d448af2, entries=150, sequenceid=46, filesize=11.7 K 2024-11-24T08:13:44,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=53.67 KB/54960 for 187d8f78b18a17144230e52cfd716831 in 1068ms, sequenceid=46, compaction requested=false 2024-11-24T08:13:44,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:44,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-24T08:13:44,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:44,165 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T08:13:44,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:44,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:44,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:44,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:44,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:44,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/eb8543c9c0e74e36a9f70b3bd3211e8f is 50, key is test_row_1/A:col10/1732436023117/Put/seqid=0 2024-11-24T08:13:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T08:13:44,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741843_1019 (size=9657) 2024-11-24T08:13:44,226 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/eb8543c9c0e74e36a9f70b3bd3211e8f 2024-11-24T08:13:44,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d46077d896084e0a9097d89a85c4750b is 50, key is test_row_1/B:col10/1732436023117/Put/seqid=0 2024-11-24T08:13:44,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741844_1020 (size=9657) 2024-11-24T08:13:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:44,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:44,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436084435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436024355, max timestamp=1732436024444Num uniques 859; 2024-11-24T08:13:44,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:13:44,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732436024355, max timestamp=1732436024444Num uniques 859; 2024-11-24T08:13:44,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-24T08:13:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732436024355, max timestamp=1732436024444Num uniques 859; 2024-11-24T08:13:44,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-24T08:13:44,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436084513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436084515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436084516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436084521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,529 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436084560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436084631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436084632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436084631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436084641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,685 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d46077d896084e0a9097d89a85c4750b 2024-11-24T08:13:44,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/7afe45f68a414bb5afcb2730097ea1cb is 50, key is test_row_1/C:col10/1732436023117/Put/seqid=0 2024-11-24T08:13:44,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741845_1021 (size=9657) 2024-11-24T08:13:44,723 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=57 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/7afe45f68a414bb5afcb2730097ea1cb 2024-11-24T08:13:44,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/eb8543c9c0e74e36a9f70b3bd3211e8f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/eb8543c9c0e74e36a9f70b3bd3211e8f 2024-11-24T08:13:44,754 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/eb8543c9c0e74e36a9f70b3bd3211e8f, entries=100, sequenceid=57, filesize=9.4 K 2024-11-24T08:13:44,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d46077d896084e0a9097d89a85c4750b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d46077d896084e0a9097d89a85c4750b 2024-11-24T08:13:44,768 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d46077d896084e0a9097d89a85c4750b, entries=100, sequenceid=57, filesize=9.4 K 2024-11-24T08:13:44,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/7afe45f68a414bb5afcb2730097ea1cb as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/7afe45f68a414bb5afcb2730097ea1cb 2024-11-24T08:13:44,775 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T08:13:44,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436084770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,785 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/7afe45f68a414bb5afcb2730097ea1cb, entries=100, sequenceid=57, filesize=9.4 K 2024-11-24T08:13:44,789 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=126.68 KB/129720 for 187d8f78b18a17144230e52cfd716831 in 624ms, sequenceid=57, compaction requested=true 2024-11-24T08:13:44,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:44,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:44,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-24T08:13:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-24T08:13:44,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T08:13:44,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7310 sec 2024-11-24T08:13:44,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.7490 sec 2024-11-24T08:13:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:44,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=133.39 KB heapSize=383.16 KB 2024-11-24T08:13:44,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732436024507, max timestamp=1732436024843Num uniques 1018; 2024-11-24T08:13:44,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T08:13:44,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:44,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:13:44,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:44,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T08:13:44,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:44,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T08:13:44,853 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T08:13:44,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/7bba3f577e7c48e1ac055e4c390b08ed is 50, key is test_row_0/A:col10/1732436024518/Put/seqid=0 2024-11-24T08:13:44,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436084887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436084886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436084896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436084909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:44,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:44,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741846_1022 (size=16681) 2024-11-24T08:13:45,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436085011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436085011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436085020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436085021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436085090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436085220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436085221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436085230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436085240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/7bba3f577e7c48e1ac055e4c390b08ed 2024-11-24T08:13:45,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/e8d0f35d441543918b4839c2d59d1747 is 50, key is test_row_0/B:col10/1732436024518/Put/seqid=0 2024-11-24T08:13:45,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741847_1023 (size=12001) 2024-11-24T08:13:45,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/e8d0f35d441543918b4839c2d59d1747 2024-11-24T08:13:45,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/9734c4687f84450c8a51be381cc675ce is 50, key is test_row_0/C:col10/1732436024518/Put/seqid=0 2024-11-24T08:13:45,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741848_1024 (size=12001) 2024-11-24T08:13:45,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/9734c4687f84450c8a51be381cc675ce 2024-11-24T08:13:45,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/7bba3f577e7c48e1ac055e4c390b08ed as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/7bba3f577e7c48e1ac055e4c390b08ed 2024-11-24T08:13:45,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/7bba3f577e7c48e1ac055e4c390b08ed, entries=250, sequenceid=95, filesize=16.3 K 2024-11-24T08:13:45,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/e8d0f35d441543918b4839c2d59d1747 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e8d0f35d441543918b4839c2d59d1747 2024-11-24T08:13:45,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e8d0f35d441543918b4839c2d59d1747, entries=150, sequenceid=95, filesize=11.7 K 2024-11-24T08:13:45,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/9734c4687f84450c8a51be381cc675ce as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/9734c4687f84450c8a51be381cc675ce 2024-11-24T08:13:45,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/9734c4687f84450c8a51be381cc675ce, entries=150, sequenceid=95, filesize=11.7 K 2024-11-24T08:13:45,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~104.32 KB/106820, heapSize ~295.41 KB/302496, currentSize=80.51 KB/82440 for 187d8f78b18a17144230e52cfd716831 in 687ms, sequenceid=95, compaction requested=true 2024-11-24T08:13:45,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:45,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:13:45,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:45,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:13:45,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:45,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:13:45,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:45,555 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T08:13:45,555 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:13:45,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:45,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:45,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:45,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:45,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:45,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:45,557 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:13:45,560 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:13:45,562 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52680 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:13:45,562 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/B is initiating minor compaction (all files) 2024-11-24T08:13:45,562 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/A is initiating minor compaction (all files) 2024-11-24T08:13:45,563 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/B in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:45,563 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/A in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:45,563 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/807a070f6b2b47759cde355f70382322, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e75ad5ead35d47d9ab581af4726ead02, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d46077d896084e0a9097d89a85c4750b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e8d0f35d441543918b4839c2d59d1747] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=44.6 K 2024-11-24T08:13:45,563 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/523bef2d65a24f40a97c00547145caad, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/c9ac77af26514e0a845fdb38abd37deb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/eb8543c9c0e74e36a9f70b3bd3211e8f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/7bba3f577e7c48e1ac055e4c390b08ed] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=51.4 K 2024-11-24T08:13:45,565 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 523bef2d65a24f40a97c00547145caad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732436022048 2024-11-24T08:13:45,566 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 807a070f6b2b47759cde355f70382322, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732436022048 2024-11-24T08:13:45,566 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9ac77af26514e0a845fdb38abd37deb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732436022346 2024-11-24T08:13:45,567 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting e75ad5ead35d47d9ab581af4726ead02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732436022346 2024-11-24T08:13:45,568 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb8543c9c0e74e36a9f70b3bd3211e8f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732436023117 2024-11-24T08:13:45,568 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting d46077d896084e0a9097d89a85c4750b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732436023117 2024-11-24T08:13:45,569 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bba3f577e7c48e1ac055e4c390b08ed, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732436024507 2024-11-24T08:13:45,569 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting e8d0f35d441543918b4839c2d59d1747, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732436024518 2024-11-24T08:13:45,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/5b2e7a5326384794b91ed863ba671f26 is 50, key is test_row_0/A:col10/1732436025548/Put/seqid=0 2024-11-24T08:13:45,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436085595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436085600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,619 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#A#compaction#13 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:45,620 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/ccc32d4b2d6c4eaca4f7f7793d2d279c is 50, key is test_row_0/A:col10/1732436024518/Put/seqid=0 2024-11-24T08:13:45,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436085605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436085608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741849_1025 (size=14341) 2024-11-24T08:13:45,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/5b2e7a5326384794b91ed863ba671f26 2024-11-24T08:13:45,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436085610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,649 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#B#compaction#14 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:45,650 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/e9f00be65a8041cb896463a331587549 is 50, key is test_row_0/B:col10/1732436024518/Put/seqid=0 2024-11-24T08:13:45,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/ca1484b40989425a92e47fc903184027 is 50, key is test_row_0/B:col10/1732436025548/Put/seqid=0 2024-11-24T08:13:45,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741850_1026 (size=12139) 2024-11-24T08:13:45,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741851_1027 (size=12139) 2024-11-24T08:13:45,680 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/ccc32d4b2d6c4eaca4f7f7793d2d279c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/ccc32d4b2d6c4eaca4f7f7793d2d279c 2024-11-24T08:13:45,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741852_1028 (size=12001) 2024-11-24T08:13:45,705 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 187d8f78b18a17144230e52cfd716831/A of 187d8f78b18a17144230e52cfd716831 into ccc32d4b2d6c4eaca4f7f7793d2d279c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:45,705 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:45,708 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/A, priority=12, startTime=1732436025534; duration=0sec 2024-11-24T08:13:45,709 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:45,709 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:A 2024-11-24T08:13:45,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/ca1484b40989425a92e47fc903184027 2024-11-24T08:13:45,711 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:13:45,720 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:13:45,721 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/C is initiating minor compaction (all files) 2024-11-24T08:13:45,721 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/C in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:45,721 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/0762a25cddd54f39a2c59058e5733b04, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/120c75f2771d4716a95baa0e3d448af2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/7afe45f68a414bb5afcb2730097ea1cb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/9734c4687f84450c8a51be381cc675ce] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=44.6 K 2024-11-24T08:13:45,723 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0762a25cddd54f39a2c59058e5733b04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732436022048 2024-11-24T08:13:45,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436085720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,727 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 120c75f2771d4716a95baa0e3d448af2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732436022346 2024-11-24T08:13:45,728 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7afe45f68a414bb5afcb2730097ea1cb, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=57, earliestPutTs=1732436023117 2024-11-24T08:13:45,729 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9734c4687f84450c8a51be381cc675ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732436024518 2024-11-24T08:13:45,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/a6958a591991450c9f542afe99280680 is 50, key is test_row_0/C:col10/1732436025548/Put/seqid=0 2024-11-24T08:13:45,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436085730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436085730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436085741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:45,766 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#C#compaction#17 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:45,767 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/50350840f9004f0dbec40525a5398c70 is 50, key is test_row_0/C:col10/1732436024518/Put/seqid=0 2024-11-24T08:13:45,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741853_1029 (size=12001) 2024-11-24T08:13:45,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/a6958a591991450c9f542afe99280680 2024-11-24T08:13:45,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/5b2e7a5326384794b91ed863ba671f26 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/5b2e7a5326384794b91ed863ba671f26 2024-11-24T08:13:45,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/5b2e7a5326384794b91ed863ba671f26, entries=200, sequenceid=112, filesize=14.0 K 2024-11-24T08:13:45,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/ca1484b40989425a92e47fc903184027 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/ca1484b40989425a92e47fc903184027 2024-11-24T08:13:45,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741854_1030 (size=12139) 2024-11-24T08:13:45,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/ca1484b40989425a92e47fc903184027, entries=150, sequenceid=112, filesize=11.7 K 2024-11-24T08:13:45,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/a6958a591991450c9f542afe99280680 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/a6958a591991450c9f542afe99280680 2024-11-24T08:13:45,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/a6958a591991450c9f542afe99280680, entries=150, sequenceid=112, filesize=11.7 K 2024-11-24T08:13:45,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 187d8f78b18a17144230e52cfd716831 in 276ms, sequenceid=112, compaction requested=false 2024-11-24T08:13:45,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:45,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:45,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T08:13:45,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:45,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:45,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:45,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:45,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:45,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:46,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/cf6faadb4d654bc3a6c824719adaf225 is 50, key is test_row_0/A:col10/1732436025957/Put/seqid=0 2024-11-24T08:13:46,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741855_1031 (size=12151) 2024-11-24T08:13:46,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/cf6faadb4d654bc3a6c824719adaf225 2024-11-24T08:13:46,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436086030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436086027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436086032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436086041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/1f8ffe5a520b4c7c883d967a02b6825b is 50, key is test_row_0/B:col10/1732436025957/Put/seqid=0 2024-11-24T08:13:46,087 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/e9f00be65a8041cb896463a331587549 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e9f00be65a8041cb896463a331587549 2024-11-24T08:13:46,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741856_1032 (size=12151) 2024-11-24T08:13:46,099 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 187d8f78b18a17144230e52cfd716831/B of 187d8f78b18a17144230e52cfd716831 into e9f00be65a8041cb896463a331587549(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:46,099 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:46,100 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/B, priority=12, startTime=1732436025554; duration=0sec 2024-11-24T08:13:46,100 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:46,100 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:B 2024-11-24T08:13:46,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436086151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436086151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436086151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436086160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-24T08:13:46,220 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/50350840f9004f0dbec40525a5398c70 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/50350840f9004f0dbec40525a5398c70 2024-11-24T08:13:46,221 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:13:46,221 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:13:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-24T08:13:46,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T08:13:46,229 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:13:46,230 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:13:46,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:13:46,240 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 187d8f78b18a17144230e52cfd716831/C of 187d8f78b18a17144230e52cfd716831 into 50350840f9004f0dbec40525a5398c70(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:46,240 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:46,241 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/C, priority=12, startTime=1732436025554; duration=0sec 2024-11-24T08:13:46,241 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:46,241 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:C 2024-11-24T08:13:46,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T08:13:46,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T08:13:46,303 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T08:13:46,303 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T08:13:46,304 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T08:13:46,304 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T08:13:46,306 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T08:13:46,306 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-24T08:13:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T08:13:46,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436086360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436086360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436086361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436086370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T08:13:46,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:46,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:46,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:46,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:46,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:46,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/1f8ffe5a520b4c7c883d967a02b6825b 2024-11-24T08:13:46,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/5cff8a1835b747af8c9184fe2e85eda5 is 50, key is test_row_0/C:col10/1732436025957/Put/seqid=0 2024-11-24T08:13:46,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T08:13:46,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:46,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:46,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:46,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:46,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:46,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741857_1033 (size=12151) 2024-11-24T08:13:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T08:13:46,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/5cff8a1835b747af8c9184fe2e85eda5 2024-11-24T08:13:46,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/cf6faadb4d654bc3a6c824719adaf225 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/cf6faadb4d654bc3a6c824719adaf225 2024-11-24T08:13:46,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/cf6faadb4d654bc3a6c824719adaf225, entries=150, sequenceid=131, filesize=11.9 K 2024-11-24T08:13:46,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/1f8ffe5a520b4c7c883d967a02b6825b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/1f8ffe5a520b4c7c883d967a02b6825b 2024-11-24T08:13:46,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/1f8ffe5a520b4c7c883d967a02b6825b, entries=150, sequenceid=131, filesize=11.9 K 2024-11-24T08:13:46,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/5cff8a1835b747af8c9184fe2e85eda5 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/5cff8a1835b747af8c9184fe2e85eda5 2024-11-24T08:13:46,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/5cff8a1835b747af8c9184fe2e85eda5, entries=150, sequenceid=131, filesize=11.9 K 2024-11-24T08:13:46,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 187d8f78b18a17144230e52cfd716831 in 615ms, sequenceid=131, compaction requested=true 2024-11-24T08:13:46,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:46,606 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:46,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:13:46,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:46,607 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:46,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:13:46,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:46,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:13:46,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:46,612 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:46,612 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38631 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:46,612 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/B is initiating minor compaction (all files) 2024-11-24T08:13:46,612 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/A is initiating minor compaction (all files) 2024-11-24T08:13:46,612 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/B in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:46,612 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/A in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:46,612 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e9f00be65a8041cb896463a331587549, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/ca1484b40989425a92e47fc903184027, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/1f8ffe5a520b4c7c883d967a02b6825b] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=35.4 K 2024-11-24T08:13:46,613 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/ccc32d4b2d6c4eaca4f7f7793d2d279c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/5b2e7a5326384794b91ed863ba671f26, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/cf6faadb4d654bc3a6c824719adaf225] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=37.7 K 2024-11-24T08:13:46,614 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting e9f00be65a8041cb896463a331587549, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732436024518 2024-11-24T08:13:46,615 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccc32d4b2d6c4eaca4f7f7793d2d279c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732436024518 2024-11-24T08:13:46,616 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b2e7a5326384794b91ed863ba671f26, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732436024862 2024-11-24T08:13:46,616 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting ca1484b40989425a92e47fc903184027, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732436024893 2024-11-24T08:13:46,618 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf6faadb4d654bc3a6c824719adaf225, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732436025932 2024-11-24T08:13:46,618 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f8ffe5a520b4c7c883d967a02b6825b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732436025932 2024-11-24T08:13:46,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:46,631 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T08:13:46,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:46,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:46,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:46,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:46,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:46,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:46,661 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#A#compaction#21 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:46,662 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/85389a0a5c88400d967b4e394a12f0d1 is 50, key is test_row_0/A:col10/1732436025957/Put/seqid=0 2024-11-24T08:13:46,667 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#B#compaction#22 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:46,668 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/7a8e3b470b5b404d8a120e97ff6cb7ed is 50, key is test_row_0/B:col10/1732436025957/Put/seqid=0 2024-11-24T08:13:46,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/8a5ad919770c46018e2aac499190ee1a is 50, key is test_row_0/A:col10/1732436026003/Put/seqid=0 2024-11-24T08:13:46,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T08:13:46,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:46,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:46,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:46,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:46,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:46,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436086695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436086697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436086708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436086710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436086713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741858_1034 (size=12391) 2024-11-24T08:13:46,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741859_1035 (size=12391) 2024-11-24T08:13:46,755 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/85389a0a5c88400d967b4e394a12f0d1 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/85389a0a5c88400d967b4e394a12f0d1 2024-11-24T08:13:46,763 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/7a8e3b470b5b404d8a120e97ff6cb7ed as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/7a8e3b470b5b404d8a120e97ff6cb7ed 2024-11-24T08:13:46,766 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/A of 187d8f78b18a17144230e52cfd716831 into 85389a0a5c88400d967b4e394a12f0d1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:46,766 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:46,766 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/A, priority=13, startTime=1732436026606; duration=0sec 2024-11-24T08:13:46,766 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:46,766 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:A 2024-11-24T08:13:46,766 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:46,770 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:46,770 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/C is initiating minor compaction (all files) 2024-11-24T08:13:46,770 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/C in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:46,770 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/50350840f9004f0dbec40525a5398c70, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/a6958a591991450c9f542afe99280680, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/5cff8a1835b747af8c9184fe2e85eda5] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=35.4 K 2024-11-24T08:13:46,771 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50350840f9004f0dbec40525a5398c70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732436024518 2024-11-24T08:13:46,772 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6958a591991450c9f542afe99280680, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732436024893 2024-11-24T08:13:46,773 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cff8a1835b747af8c9184fe2e85eda5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732436025932 2024-11-24T08:13:46,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741860_1036 (size=12151) 2024-11-24T08:13:46,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/8a5ad919770c46018e2aac499190ee1a 2024-11-24T08:13:46,777 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/B of 187d8f78b18a17144230e52cfd716831 into 7a8e3b470b5b404d8a120e97ff6cb7ed(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:46,777 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:46,777 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/B, priority=13, startTime=1732436026607; duration=0sec 2024-11-24T08:13:46,777 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:46,777 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:B 2024-11-24T08:13:46,793 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:46,794 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/1d6edc1919a041aeb50510aeef2fc32f is 50, key is test_row_0/C:col10/1732436025957/Put/seqid=0 2024-11-24T08:13:46,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/874910523ff141418535f614b5c7a654 is 50, key is test_row_0/B:col10/1732436026003/Put/seqid=0 2024-11-24T08:13:46,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741861_1037 (size=12391) 2024-11-24T08:13:46,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436086820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436086821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436086821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436086821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:46,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436086821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:46,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:46,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:46,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741862_1038 (size=12151) 2024-11-24T08:13:46,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/874910523ff141418535f614b5c7a654 2024-11-24T08:13:46,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/ef699736767d4c82b0322e968d8c199d is 50, key is test_row_0/C:col10/1732436026003/Put/seqid=0 2024-11-24T08:13:46,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T08:13:46,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:46,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:46,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:46,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:46,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:46,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:46,858 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38469 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=30c28c82771d,45087,1732436016722, table=TestAcidGuarantees, region=187d8f78b18a17144230e52cfd716831. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T08:13:46,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T08:13:46,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741863_1039 (size=12151) 2024-11-24T08:13:46,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/ef699736767d4c82b0322e968d8c199d 2024-11-24T08:13:46,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/8a5ad919770c46018e2aac499190ee1a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/8a5ad919770c46018e2aac499190ee1a 2024-11-24T08:13:46,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/8a5ad919770c46018e2aac499190ee1a, entries=150, sequenceid=148, filesize=11.9 K 2024-11-24T08:13:46,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/874910523ff141418535f614b5c7a654 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/874910523ff141418535f614b5c7a654 2024-11-24T08:13:46,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/874910523ff141418535f614b5c7a654, entries=150, sequenceid=148, filesize=11.9 K 2024-11-24T08:13:46,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/ef699736767d4c82b0322e968d8c199d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ef699736767d4c82b0322e968d8c199d 2024-11-24T08:13:46,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ef699736767d4c82b0322e968d8c199d, entries=150, sequenceid=148, filesize=11.9 K 2024-11-24T08:13:46,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 187d8f78b18a17144230e52cfd716831 in 315ms, sequenceid=148, compaction requested=false 2024-11-24T08:13:46,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:47,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-24T08:13:47,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:47,009 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T08:13:47,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:47,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:47,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:47,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:47,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:47,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:47,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/5c17b42c6b834a33be8e98f71223a54b is 50, key is test_row_0/A:col10/1732436026705/Put/seqid=0 2024-11-24T08:13:47,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741864_1040 (size=12151) 2024-11-24T08:13:47,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:47,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:47,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436087057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436087057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436087059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436087060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436087062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436087170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436087171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436087172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436087173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436087173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,231 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/1d6edc1919a041aeb50510aeef2fc32f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/1d6edc1919a041aeb50510aeef2fc32f 2024-11-24T08:13:47,247 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/C of 187d8f78b18a17144230e52cfd716831 into 1d6edc1919a041aeb50510aeef2fc32f(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:47,247 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:47,247 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/C, priority=13, startTime=1732436026611; duration=0sec 2024-11-24T08:13:47,248 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:47,249 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:C 2024-11-24T08:13:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T08:13:47,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436087381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436087381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436087382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436087382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436087390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,393 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,428 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/5c17b42c6b834a33be8e98f71223a54b 2024-11-24T08:13:47,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/c9f0380887dc414c9e22f77d0cf8d237 is 50, key is test_row_0/B:col10/1732436026705/Put/seqid=0 2024-11-24T08:13:47,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741865_1041 (size=12151) 2024-11-24T08:13:47,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436087702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436087702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436087702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436087702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436087702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:47,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:47,892 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/c9f0380887dc414c9e22f77d0cf8d237 2024-11-24T08:13:47,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/f5759d0aeb7049229dfd0521be5aaf93 is 50, key is test_row_0/C:col10/1732436026705/Put/seqid=0 2024-11-24T08:13:47,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741866_1042 (size=12151) 2024-11-24T08:13:47,919 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/f5759d0aeb7049229dfd0521be5aaf93 2024-11-24T08:13:47,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/5c17b42c6b834a33be8e98f71223a54b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/5c17b42c6b834a33be8e98f71223a54b 2024-11-24T08:13:47,944 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/5c17b42c6b834a33be8e98f71223a54b, entries=150, sequenceid=167, filesize=11.9 K 2024-11-24T08:13:47,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/c9f0380887dc414c9e22f77d0cf8d237 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/c9f0380887dc414c9e22f77d0cf8d237 2024-11-24T08:13:47,955 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/c9f0380887dc414c9e22f77d0cf8d237, entries=150, sequenceid=167, filesize=11.9 K 2024-11-24T08:13:47,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/f5759d0aeb7049229dfd0521be5aaf93 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/f5759d0aeb7049229dfd0521be5aaf93 2024-11-24T08:13:47,965 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/f5759d0aeb7049229dfd0521be5aaf93, entries=150, sequenceid=167, filesize=11.9 K 2024-11-24T08:13:47,971 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 187d8f78b18a17144230e52cfd716831 in 963ms, sequenceid=167, compaction requested=true 2024-11-24T08:13:47,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:47,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:47,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-24T08:13:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-24T08:13:47,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-24T08:13:47,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7440 sec 2024-11-24T08:13:47,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.7580 sec 2024-11-24T08:13:48,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T08:13:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:48,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:48,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:48,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:48,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:48,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:48,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:48,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/806c87ece1d04c029a60e5ff2554b6e4 is 50, key is test_row_0/A:col10/1732436028215/Put/seqid=0 2024-11-24T08:13:48,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436088244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436088247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436088247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436088249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436088256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741867_1043 (size=12151) 2024-11-24T08:13:48,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436088360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436088370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436088370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436088371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-24T08:13:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,382 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:13:48,382 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:13:48,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436088381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-24T08:13:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T08:13:48,386 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:13:48,388 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:13:48,388 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:13:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T08:13:48,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T08:13:48,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:48,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:48,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:48,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:48,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:48,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436088570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436088590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436088590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436088590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436088590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/806c87ece1d04c029a60e5ff2554b6e4 2024-11-24T08:13:48,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/758bf47897ca463799a60de1701d5f88 is 50, key is test_row_0/B:col10/1732436028215/Put/seqid=0 2024-11-24T08:13:48,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T08:13:48,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:48,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:48,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:48,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:48,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:48,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T08:13:48,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741868_1044 (size=12151) 2024-11-24T08:13:48,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/758bf47897ca463799a60de1701d5f88 2024-11-24T08:13:48,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/ba753cddbc9e48aab31a8a01a7efc9af is 50, key is test_row_0/C:col10/1732436028215/Put/seqid=0 2024-11-24T08:13:48,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741869_1045 (size=12151) 2024-11-24T08:13:48,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T08:13:48,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:48,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:48,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:48,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:48,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:48,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436088881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,884 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436088900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436088910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436088910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:48,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436088911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:48,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:48,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T08:13:49,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:49,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:49,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:49,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:49,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T08:13:49,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T08:13:49,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:49,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/ba753cddbc9e48aab31a8a01a7efc9af 2024-11-24T08:13:49,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:49,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:49,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:49,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:49,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:49,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/806c87ece1d04c029a60e5ff2554b6e4 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/806c87ece1d04c029a60e5ff2554b6e4 2024-11-24T08:13:49,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/806c87ece1d04c029a60e5ff2554b6e4, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T08:13:49,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/758bf47897ca463799a60de1701d5f88 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/758bf47897ca463799a60de1701d5f88 2024-11-24T08:13:49,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/758bf47897ca463799a60de1701d5f88, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T08:13:49,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/ba753cddbc9e48aab31a8a01a7efc9af as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ba753cddbc9e48aab31a8a01a7efc9af 2024-11-24T08:13:49,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ba753cddbc9e48aab31a8a01a7efc9af, entries=150, sequenceid=186, filesize=11.9 K 2024-11-24T08:13:49,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 187d8f78b18a17144230e52cfd716831 in 1001ms, sequenceid=186, compaction requested=true 2024-11-24T08:13:49,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:49,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:13:49,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:49,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:13:49,221 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:13:49,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:49,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:13:49,221 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:13:49,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:49,223 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:13:49,223 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/A is initiating minor compaction (all files) 2024-11-24T08:13:49,224 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/A in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:49,224 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/85389a0a5c88400d967b4e394a12f0d1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/8a5ad919770c46018e2aac499190ee1a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/5c17b42c6b834a33be8e98f71223a54b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/806c87ece1d04c029a60e5ff2554b6e4] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=47.7 K 2024-11-24T08:13:49,225 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:13:49,225 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/B is initiating minor compaction (all files) 2024-11-24T08:13:49,225 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/B in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:49,225 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/7a8e3b470b5b404d8a120e97ff6cb7ed, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/874910523ff141418535f614b5c7a654, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/c9f0380887dc414c9e22f77d0cf8d237, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/758bf47897ca463799a60de1701d5f88] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=47.7 K 2024-11-24T08:13:49,225 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85389a0a5c88400d967b4e394a12f0d1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732436025932 2024-11-24T08:13:49,226 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a8e3b470b5b404d8a120e97ff6cb7ed, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732436025932 2024-11-24T08:13:49,226 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a5ad919770c46018e2aac499190ee1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732436026003 2024-11-24T08:13:49,227 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 874910523ff141418535f614b5c7a654, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732436026003 2024-11-24T08:13:49,228 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c17b42c6b834a33be8e98f71223a54b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732436026705 2024-11-24T08:13:49,228 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 806c87ece1d04c029a60e5ff2554b6e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732436027058 2024-11-24T08:13:49,228 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting c9f0380887dc414c9e22f77d0cf8d237, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732436026705 2024-11-24T08:13:49,229 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 758bf47897ca463799a60de1701d5f88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732436027058 2024-11-24T08:13:49,263 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#A#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:49,265 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/bee96a9393d34b3b8d50c50762023d6f is 50, key is test_row_0/A:col10/1732436028215/Put/seqid=0 2024-11-24T08:13:49,271 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#B#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:49,271 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/5f974d2d2c994dabbd487bbdd92aad4d is 50, key is test_row_0/B:col10/1732436028215/Put/seqid=0 2024-11-24T08:13:49,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741870_1046 (size=12527) 2024-11-24T08:13:49,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741871_1047 (size=12527) 2024-11-24T08:13:49,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-24T08:13:49,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:49,322 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T08:13:49,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:49,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:49,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:49,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:49,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:49,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:49,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/6dd4866554f4490caff85f41a6602d5b is 50, key is test_row_0/A:col10/1732436028236/Put/seqid=0 2024-11-24T08:13:49,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741872_1048 (size=12151) 2024-11-24T08:13:49,351 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/6dd4866554f4490caff85f41a6602d5b 2024-11-24T08:13:49,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/4b93c33434874973b2f629f5da0bd678 is 50, key is test_row_0/B:col10/1732436028236/Put/seqid=0 2024-11-24T08:13:49,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:49,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:49,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741873_1049 (size=12151) 2024-11-24T08:13:49,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436089439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436089440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436089442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436089444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436089445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T08:13:49,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436089550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436089560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436089561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436089561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,566 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436089561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,700 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/bee96a9393d34b3b8d50c50762023d6f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/bee96a9393d34b3b8d50c50762023d6f 2024-11-24T08:13:49,707 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/5f974d2d2c994dabbd487bbdd92aad4d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5f974d2d2c994dabbd487bbdd92aad4d 2024-11-24T08:13:49,714 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 187d8f78b18a17144230e52cfd716831/A of 187d8f78b18a17144230e52cfd716831 into bee96a9393d34b3b8d50c50762023d6f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:49,714 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:49,714 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/A, priority=12, startTime=1732436029221; duration=0sec 2024-11-24T08:13:49,714 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:49,714 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:A 2024-11-24T08:13:49,714 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:13:49,719 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:13:49,719 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/C is initiating minor compaction (all files) 2024-11-24T08:13:49,719 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/C in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:49,720 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/1d6edc1919a041aeb50510aeef2fc32f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ef699736767d4c82b0322e968d8c199d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/f5759d0aeb7049229dfd0521be5aaf93, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ba753cddbc9e48aab31a8a01a7efc9af] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=47.7 K 2024-11-24T08:13:49,720 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d6edc1919a041aeb50510aeef2fc32f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732436025932 2024-11-24T08:13:49,721 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef699736767d4c82b0322e968d8c199d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732436026003 2024-11-24T08:13:49,721 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 187d8f78b18a17144230e52cfd716831/B of 187d8f78b18a17144230e52cfd716831 into 5f974d2d2c994dabbd487bbdd92aad4d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:49,721 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:49,722 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/B, priority=12, startTime=1732436029221; duration=0sec 2024-11-24T08:13:49,722 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5759d0aeb7049229dfd0521be5aaf93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732436026705 2024-11-24T08:13:49,722 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:49,722 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:B 2024-11-24T08:13:49,722 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba753cddbc9e48aab31a8a01a7efc9af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732436027058 2024-11-24T08:13:49,745 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#C#compaction#37 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:49,746 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/6caf7e4527a94e7ca6037ecbbde35b6a is 50, key is test_row_0/C:col10/1732436028215/Put/seqid=0 2024-11-24T08:13:49,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741874_1050 (size=12527) 2024-11-24T08:13:49,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436089760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436089770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436089770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436089770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436089770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,777 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:49,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:49,811 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/4b93c33434874973b2f629f5da0bd678 2024-11-24T08:13:49,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/cc7348022d9f4b6781ef69cce6c7ecff is 50, key is test_row_0/C:col10/1732436028236/Put/seqid=0 2024-11-24T08:13:49,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741875_1051 (size=12151) 2024-11-24T08:13:49,829 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/cc7348022d9f4b6781ef69cce6c7ecff 2024-11-24T08:13:49,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/6dd4866554f4490caff85f41a6602d5b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/6dd4866554f4490caff85f41a6602d5b 2024-11-24T08:13:49,842 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/6dd4866554f4490caff85f41a6602d5b, entries=150, sequenceid=200, filesize=11.9 K 2024-11-24T08:13:49,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/4b93c33434874973b2f629f5da0bd678 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/4b93c33434874973b2f629f5da0bd678 2024-11-24T08:13:49,850 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/4b93c33434874973b2f629f5da0bd678, entries=150, sequenceid=200, filesize=11.9 K 2024-11-24T08:13:49,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/cc7348022d9f4b6781ef69cce6c7ecff as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/cc7348022d9f4b6781ef69cce6c7ecff 2024-11-24T08:13:49,858 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/cc7348022d9f4b6781ef69cce6c7ecff, entries=150, sequenceid=200, filesize=11.9 K 2024-11-24T08:13:49,859 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 187d8f78b18a17144230e52cfd716831 in 537ms, sequenceid=200, compaction requested=false 2024-11-24T08:13:49,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:49,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:49,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-24T08:13:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-24T08:13:49,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-24T08:13:49,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4730 sec 2024-11-24T08:13:49,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.4840 sec 2024-11-24T08:13:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:50,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T08:13:50,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:50,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:50,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:50,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:50,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:50,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:50,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/aa629b9b67484096946fc0c49d4bfa40 is 50, key is test_row_0/A:col10/1732436029442/Put/seqid=0 2024-11-24T08:13:50,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741876_1052 (size=14541) 2024-11-24T08:13:50,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/aa629b9b67484096946fc0c49d4bfa40 2024-11-24T08:13:50,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/fedf639e66ff4f17a2f8d8d4a400b094 is 50, key is test_row_0/B:col10/1732436029442/Put/seqid=0 2024-11-24T08:13:50,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741877_1053 (size=12151) 2024-11-24T08:13:50,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436090138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436090140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436090139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436090143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436090145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,172 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/6caf7e4527a94e7ca6037ecbbde35b6a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/6caf7e4527a94e7ca6037ecbbde35b6a 2024-11-24T08:13:50,181 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 187d8f78b18a17144230e52cfd716831/C of 187d8f78b18a17144230e52cfd716831 into 6caf7e4527a94e7ca6037ecbbde35b6a(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:50,181 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:50,181 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/C, priority=12, startTime=1732436029221; duration=0sec 2024-11-24T08:13:50,181 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:50,181 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:C 2024-11-24T08:13:50,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436090250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436090250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436090251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436090251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436090251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436090460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436090460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436090460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436090470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436090470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/fedf639e66ff4f17a2f8d8d4a400b094 2024-11-24T08:13:50,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/e4a22a696857466c89e3709a52a3f5b7 is 50, key is test_row_0/C:col10/1732436029442/Put/seqid=0 2024-11-24T08:13:50,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741878_1054 (size=12151) 2024-11-24T08:13:50,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-24T08:13:50,530 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:13:50,530 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:13:50,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-24T08:13:50,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T08:13:50,533 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:13:50,534 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:13:50,534 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:13:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T08:13:50,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T08:13:50,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:50,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:50,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:50,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:50,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:50,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:50,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436090769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436090770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436090770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436090780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436090780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:50,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T08:13:50,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:50,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:50,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:50,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:50,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:50,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T08:13:50,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/e4a22a696857466c89e3709a52a3f5b7 2024-11-24T08:13:50,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/aa629b9b67484096946fc0c49d4bfa40 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/aa629b9b67484096946fc0c49d4bfa40 2024-11-24T08:13:50,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/aa629b9b67484096946fc0c49d4bfa40, entries=200, sequenceid=221, filesize=14.2 K 2024-11-24T08:13:50,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/fedf639e66ff4f17a2f8d8d4a400b094 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/fedf639e66ff4f17a2f8d8d4a400b094 2024-11-24T08:13:50,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/fedf639e66ff4f17a2f8d8d4a400b094, entries=150, sequenceid=221, filesize=11.9 K 2024-11-24T08:13:50,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/e4a22a696857466c89e3709a52a3f5b7 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/e4a22a696857466c89e3709a52a3f5b7 2024-11-24T08:13:50,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/e4a22a696857466c89e3709a52a3f5b7, entries=150, sequenceid=221, filesize=11.9 K 2024-11-24T08:13:50,962 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 187d8f78b18a17144230e52cfd716831 in 889ms, sequenceid=221, compaction requested=true 2024-11-24T08:13:50,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:50,962 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:50,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:13:50,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:50,963 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:50,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:13:50,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:50,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:13:50,963 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:50,964 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:50,964 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/A is initiating minor compaction (all files) 2024-11-24T08:13:50,964 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/A in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:50,964 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/bee96a9393d34b3b8d50c50762023d6f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/6dd4866554f4490caff85f41a6602d5b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/aa629b9b67484096946fc0c49d4bfa40] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=38.3 K 2024-11-24T08:13:50,965 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:50,965 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/B is initiating minor compaction (all files) 2024-11-24T08:13:50,965 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/B in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:50,965 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5f974d2d2c994dabbd487bbdd92aad4d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/4b93c33434874973b2f629f5da0bd678, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/fedf639e66ff4f17a2f8d8d4a400b094] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=36.0 K 2024-11-24T08:13:50,965 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting bee96a9393d34b3b8d50c50762023d6f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732436027058 2024-11-24T08:13:50,966 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f974d2d2c994dabbd487bbdd92aad4d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732436027058 2024-11-24T08:13:50,966 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dd4866554f4490caff85f41a6602d5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732436028236 2024-11-24T08:13:50,966 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b93c33434874973b2f629f5da0bd678, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732436028236 2024-11-24T08:13:50,967 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting fedf639e66ff4f17a2f8d8d4a400b094, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732436029440 2024-11-24T08:13:50,967 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa629b9b67484096946fc0c49d4bfa40, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732436029436 2024-11-24T08:13:50,980 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#A#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:50,982 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#B#compaction#43 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:50,983 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/3306454b3d3241b1b8e9fb55396e7a0f is 50, key is test_row_0/B:col10/1732436029442/Put/seqid=0 2024-11-24T08:13:50,984 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/43fc05016c5a4a9a80700e9740eb2433 is 50, key is test_row_0/A:col10/1732436029442/Put/seqid=0 2024-11-24T08:13:50,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-24T08:13:50,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:50,998 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T08:13:50,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:50,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:50,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:50,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:50,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:50,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:51,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741879_1055 (size=12629) 2024-11-24T08:13:51,015 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/43fc05016c5a4a9a80700e9740eb2433 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/43fc05016c5a4a9a80700e9740eb2433 2024-11-24T08:13:51,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741880_1056 (size=12629) 2024-11-24T08:13:51,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/8566c2f8de7c44b7b89ca55b49ad0b87 is 50, key is test_row_0/A:col10/1732436030134/Put/seqid=0 2024-11-24T08:13:51,027 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/3306454b3d3241b1b8e9fb55396e7a0f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/3306454b3d3241b1b8e9fb55396e7a0f 2024-11-24T08:13:51,027 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/A of 187d8f78b18a17144230e52cfd716831 into 43fc05016c5a4a9a80700e9740eb2433(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:51,027 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:51,027 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/A, priority=13, startTime=1732436030962; duration=0sec 2024-11-24T08:13:51,027 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:51,028 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:A 2024-11-24T08:13:51,028 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:51,031 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:51,031 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/C is initiating minor compaction (all files) 2024-11-24T08:13:51,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741881_1057 (size=12151) 2024-11-24T08:13:51,032 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/C in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:51,032 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/6caf7e4527a94e7ca6037ecbbde35b6a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/cc7348022d9f4b6781ef69cce6c7ecff, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/e4a22a696857466c89e3709a52a3f5b7] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=36.0 K 2024-11-24T08:13:51,033 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6caf7e4527a94e7ca6037ecbbde35b6a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1732436027058 2024-11-24T08:13:51,034 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc7348022d9f4b6781ef69cce6c7ecff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732436028236 2024-11-24T08:13:51,035 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4a22a696857466c89e3709a52a3f5b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732436029440 2024-11-24T08:13:51,035 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/B of 187d8f78b18a17144230e52cfd716831 into 3306454b3d3241b1b8e9fb55396e7a0f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:51,035 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:51,035 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/B, priority=13, startTime=1732436030963; duration=0sec 2024-11-24T08:13:51,035 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:51,035 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:B 2024-11-24T08:13:51,050 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:51,051 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/91b7626a6e754f9e89602a2c78b73147 is 50, key is test_row_0/C:col10/1732436029442/Put/seqid=0 2024-11-24T08:13:51,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741882_1058 (size=12629) 2024-11-24T08:13:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T08:13:51,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:51,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436091302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436091303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436091304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436091305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436091311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436091419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436091419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436091420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436091420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436091420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,432 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/8566c2f8de7c44b7b89ca55b49ad0b87 2024-11-24T08:13:51,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/f44bf6b320294b66bbb4927fe131aa93 is 50, key is test_row_0/B:col10/1732436030134/Put/seqid=0 2024-11-24T08:13:51,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741883_1059 (size=12151) 2024-11-24T08:13:51,469 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/91b7626a6e754f9e89602a2c78b73147 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/91b7626a6e754f9e89602a2c78b73147 2024-11-24T08:13:51,477 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/C of 187d8f78b18a17144230e52cfd716831 into 91b7626a6e754f9e89602a2c78b73147(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:51,477 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:51,477 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/C, priority=13, startTime=1732436030963; duration=0sec 2024-11-24T08:13:51,477 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:51,477 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:C 2024-11-24T08:13:51,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436091630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436091631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436091631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436091631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436091634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,636 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T08:13:51,859 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/f44bf6b320294b66bbb4927fe131aa93 2024-11-24T08:13:51,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/160c553a9da6470991f799b4906f298b is 50, key is test_row_0/C:col10/1732436030134/Put/seqid=0 2024-11-24T08:13:51,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741884_1060 (size=12151) 2024-11-24T08:13:51,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436091940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436091940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436091940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436091941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436091941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:51,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:51,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,281 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/160c553a9da6470991f799b4906f298b 2024-11-24T08:13:52,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/8566c2f8de7c44b7b89ca55b49ad0b87 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/8566c2f8de7c44b7b89ca55b49ad0b87 2024-11-24T08:13:52,295 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/8566c2f8de7c44b7b89ca55b49ad0b87, entries=150, sequenceid=235, filesize=11.9 K 2024-11-24T08:13:52,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/f44bf6b320294b66bbb4927fe131aa93 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/f44bf6b320294b66bbb4927fe131aa93 2024-11-24T08:13:52,302 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/f44bf6b320294b66bbb4927fe131aa93, entries=150, sequenceid=235, filesize=11.9 K 2024-11-24T08:13:52,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/160c553a9da6470991f799b4906f298b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/160c553a9da6470991f799b4906f298b 2024-11-24T08:13:52,309 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/160c553a9da6470991f799b4906f298b, entries=150, sequenceid=235, filesize=11.9 K 2024-11-24T08:13:52,310 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 187d8f78b18a17144230e52cfd716831 in 1312ms, sequenceid=235, compaction requested=false 2024-11-24T08:13:52,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:52,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:52,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-24T08:13:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-24T08:13:52,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T08:13:52,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7780 sec 2024-11-24T08:13:52,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7840 sec 2024-11-24T08:13:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:52,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T08:13:52,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:52,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:13:52,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:52,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:13:52,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:52,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:13:52,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/ae3edc50af0444d78801254df5d94adf is 50, key is test_row_0/A:col10/1732436031307/Put/seqid=0 2024-11-24T08:13:52,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741885_1061 (size=12251) 2024-11-24T08:13:52,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436092502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436092502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436092510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436092511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436092512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436092620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436092620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436092620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436092620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436092620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,624 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-24T08:13:52,680 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:13:52,681 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:13:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-24T08:13:52,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T08:13:52,683 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:13:52,684 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:13:52,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:13:52,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T08:13:52,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436092829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436092829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436092830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436092830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:52,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436092830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:52,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:52,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:52,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T08:13:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:52,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:52,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:52,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/ae3edc50af0444d78801254df5d94adf 2024-11-24T08:13:52,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/50a8baaab9274431a5d4ad7853133181 is 50, key is test_row_0/B:col10/1732436031307/Put/seqid=0 2024-11-24T08:13:52,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741886_1062 (size=12251) 2024-11-24T08:13:52,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T08:13:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:52,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:52,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:52,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T08:13:53,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436093140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T08:13:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436093140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:53,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:53,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:53,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436093140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436093140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436093140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,146 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/50a8baaab9274431a5d4ad7853133181 2024-11-24T08:13:53,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/e01547b102ad45099a52922ecfacdc97 is 50, key is test_row_0/C:col10/1732436031307/Put/seqid=0 2024-11-24T08:13:53,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T08:13:53,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:53,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:53,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:53,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:53,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:53,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:53,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741887_1063 (size=12251) 2024-11-24T08:13:53,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/e01547b102ad45099a52922ecfacdc97 2024-11-24T08:13:53,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T08:13:53,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/ae3edc50af0444d78801254df5d94adf as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/ae3edc50af0444d78801254df5d94adf 2024-11-24T08:13:53,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/ae3edc50af0444d78801254df5d94adf, entries=150, sequenceid=259, filesize=12.0 K 2024-11-24T08:13:53,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/50a8baaab9274431a5d4ad7853133181 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/50a8baaab9274431a5d4ad7853133181 2024-11-24T08:13:53,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/50a8baaab9274431a5d4ad7853133181, entries=150, sequenceid=259, filesize=12.0 K 2024-11-24T08:13:53,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/e01547b102ad45099a52922ecfacdc97 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/e01547b102ad45099a52922ecfacdc97 2024-11-24T08:13:53,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/e01547b102ad45099a52922ecfacdc97, entries=150, sequenceid=259, filesize=12.0 K 2024-11-24T08:13:53,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 187d8f78b18a17144230e52cfd716831 in 881ms, sequenceid=259, compaction requested=true 2024-11-24T08:13:53,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:53,335 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:53,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:13:53,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:53,335 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:13:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:13:53,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:53,340 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:53,340 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37031 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:53,340 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/A is initiating minor compaction (all files) 2024-11-24T08:13:53,340 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/B is initiating minor compaction (all files) 2024-11-24T08:13:53,340 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/A in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:53,340 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/B in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:53,340 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/3306454b3d3241b1b8e9fb55396e7a0f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/f44bf6b320294b66bbb4927fe131aa93, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/50a8baaab9274431a5d4ad7853133181] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=36.2 K 2024-11-24T08:13:53,340 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/43fc05016c5a4a9a80700e9740eb2433, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/8566c2f8de7c44b7b89ca55b49ad0b87, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/ae3edc50af0444d78801254df5d94adf] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=36.2 K 2024-11-24T08:13:53,341 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43fc05016c5a4a9a80700e9740eb2433, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732436029440 2024-11-24T08:13:53,341 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3306454b3d3241b1b8e9fb55396e7a0f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732436029440 2024-11-24T08:13:53,341 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8566c2f8de7c44b7b89ca55b49ad0b87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732436030134 2024-11-24T08:13:53,341 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting f44bf6b320294b66bbb4927fe131aa93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732436030134 2024-11-24T08:13:53,342 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 50a8baaab9274431a5d4ad7853133181, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732436031307 2024-11-24T08:13:53,342 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae3edc50af0444d78801254df5d94adf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732436031307 2024-11-24T08:13:53,354 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:53,355 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/5eb43b2e54aa44cdbd8d2da7b55c35eb is 50, key is test_row_0/B:col10/1732436031307/Put/seqid=0 2024-11-24T08:13:53,357 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#A#compaction#52 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:53,357 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/24f38815ac774296a070f21ffc7ee0ea is 50, key is test_row_0/A:col10/1732436031307/Put/seqid=0 2024-11-24T08:13:53,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741889_1065 (size=12831) 2024-11-24T08:13:53,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741888_1064 (size=12831) 2024-11-24T08:13:53,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-24T08:13:53,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:53,454 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T08:13:53,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:53,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:53,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:53,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:53,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:53,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:53,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/130f04a41a7344d6854825a8cf59fcf9 is 50, key is test_row_0/A:col10/1732436032509/Put/seqid=0 2024-11-24T08:13:53,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741890_1066 (size=12301) 2024-11-24T08:13:53,471 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/130f04a41a7344d6854825a8cf59fcf9 2024-11-24T08:13:53,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/638087b37d414a86a95c38a4078da98d is 50, key is test_row_0/B:col10/1732436032509/Put/seqid=0 2024-11-24T08:13:53,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741891_1067 (size=12301) 2024-11-24T08:13:53,499 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/638087b37d414a86a95c38a4078da98d 2024-11-24T08:13:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/ec8644cc7a1e435a89cdc4c75ba44a67 is 50, key is test_row_0/C:col10/1732436032509/Put/seqid=0 2024-11-24T08:13:53,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741892_1068 (size=12301) 2024-11-24T08:13:53,519 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/ec8644cc7a1e435a89cdc4c75ba44a67 2024-11-24T08:13:53,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/130f04a41a7344d6854825a8cf59fcf9 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/130f04a41a7344d6854825a8cf59fcf9 2024-11-24T08:13:53,535 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/130f04a41a7344d6854825a8cf59fcf9, entries=150, sequenceid=271, filesize=12.0 K 2024-11-24T08:13:53,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/638087b37d414a86a95c38a4078da98d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/638087b37d414a86a95c38a4078da98d 2024-11-24T08:13:53,547 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/638087b37d414a86a95c38a4078da98d, entries=150, sequenceid=271, filesize=12.0 K 2024-11-24T08:13:53,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/ec8644cc7a1e435a89cdc4c75ba44a67 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ec8644cc7a1e435a89cdc4c75ba44a67 2024-11-24T08:13:53,554 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ec8644cc7a1e435a89cdc4c75ba44a67, entries=150, sequenceid=271, filesize=12.0 K 2024-11-24T08:13:53,556 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 187d8f78b18a17144230e52cfd716831 in 102ms, sequenceid=271, compaction requested=true 2024-11-24T08:13:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-24T08:13:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-24T08:13:53,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-24T08:13:53,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 875 msec 2024-11-24T08:13:53,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 881 msec 2024-11-24T08:13:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:53,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:13:53,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:53,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:53,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:53,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:53,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:53,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:53,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/52c155ac8c0441f2b93d58a26f2097c1 is 50, key is test_row_0/A:col10/1732436033651/Put/seqid=0 2024-11-24T08:13:53,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741893_1069 (size=12301) 2024-11-24T08:13:53,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/52c155ac8c0441f2b93d58a26f2097c1 2024-11-24T08:13:53,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436093691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436093691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436093700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732436033679, max timestamp=1732436033702Num uniques 859; 2024-11-24T08:13:53,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T08:13:53,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d76f8e7055844870a3d50f317efe9306 is 50, key is test_row_0/B:col10/1732436033651/Put/seqid=0 2024-11-24T08:13:53,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732436033679, max timestamp=1732436033702Num uniques 859; 2024-11-24T08:13:53,726 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T08:13:53,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732436033679, max timestamp=1732436033702Num uniques 859; 2024-11-24T08:13:53,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T08:13:53,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741894_1070 (size=12301) 2024-11-24T08:13:53,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d76f8e7055844870a3d50f317efe9306 2024-11-24T08:13:53,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/a04f64d617d242bdbcd281a4210f8f59 is 50, key is test_row_0/C:col10/1732436033651/Put/seqid=0 2024-11-24T08:13:53,785 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/24f38815ac774296a070f21ffc7ee0ea as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/24f38815ac774296a070f21ffc7ee0ea 2024-11-24T08:13:53,792 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/5eb43b2e54aa44cdbd8d2da7b55c35eb as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5eb43b2e54aa44cdbd8d2da7b55c35eb 2024-11-24T08:13:53,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436093782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741895_1071 (size=12301) 2024-11-24T08:13:53,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/a04f64d617d242bdbcd281a4210f8f59 2024-11-24T08:13:53,804 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/B of 187d8f78b18a17144230e52cfd716831 into 5eb43b2e54aa44cdbd8d2da7b55c35eb(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:53,804 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:53,804 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/B, priority=13, startTime=1732436033335; duration=0sec 2024-11-24T08:13:53,806 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:53,806 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:B 2024-11-24T08:13:53,807 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:13:53,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436093792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,811 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/A of 187d8f78b18a17144230e52cfd716831 into 24f38815ac774296a070f21ffc7ee0ea(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:53,811 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:53,811 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/A, priority=13, startTime=1732436033335; duration=0sec 2024-11-24T08:13:53,812 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:53,812 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:A 2024-11-24T08:13:53,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/52c155ac8c0441f2b93d58a26f2097c1 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/52c155ac8c0441f2b93d58a26f2097c1 2024-11-24T08:13:53,814 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49332 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:13:53,814 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/C is initiating minor compaction (all files) 2024-11-24T08:13:53,814 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/C in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:53,814 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/91b7626a6e754f9e89602a2c78b73147, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/160c553a9da6470991f799b4906f298b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/e01547b102ad45099a52922ecfacdc97, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ec8644cc7a1e435a89cdc4c75ba44a67] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=48.2 K 2024-11-24T08:13:53,815 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 91b7626a6e754f9e89602a2c78b73147, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732436029440 2024-11-24T08:13:53,815 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 160c553a9da6470991f799b4906f298b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732436030134 2024-11-24T08:13:53,816 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting e01547b102ad45099a52922ecfacdc97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732436031307 2024-11-24T08:13:53,817 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting ec8644cc7a1e435a89cdc4c75ba44a67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732436032499 2024-11-24T08:13:53,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-24T08:13:53,820 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:13:53,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/52c155ac8c0441f2b93d58a26f2097c1, entries=150, sequenceid=282, filesize=12.0 K 2024-11-24T08:13:53,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436093810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,821 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,821 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:13:53,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436093812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d76f8e7055844870a3d50f317efe9306 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d76f8e7055844870a3d50f317efe9306 2024-11-24T08:13:53,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-24T08:13:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T08:13:53,824 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:13:53,825 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:13:53,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:13:53,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d76f8e7055844870a3d50f317efe9306, entries=150, sequenceid=282, filesize=12.0 K 2024-11-24T08:13:53,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/a04f64d617d242bdbcd281a4210f8f59 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/a04f64d617d242bdbcd281a4210f8f59 2024-11-24T08:13:53,845 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:53,846 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/0573adb83d4540ff96ab92540999cc39 is 50, key is test_row_0/C:col10/1732436032509/Put/seqid=0 2024-11-24T08:13:53,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/a04f64d617d242bdbcd281a4210f8f59, entries=150, sequenceid=282, filesize=12.0 K 2024-11-24T08:13:53,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436093830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 187d8f78b18a17144230e52cfd716831 in 189ms, sequenceid=282, compaction requested=true 2024-11-24T08:13:53,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:53,851 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:53,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:A, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:13:53,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:53,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:B, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:13:53,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:53,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:13:53,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T08:13:53,853 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:53,853 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/A is initiating minor compaction (all files) 2024-11-24T08:13:53,853 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/A in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:53,853 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/24f38815ac774296a070f21ffc7ee0ea, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/130f04a41a7344d6854825a8cf59fcf9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/52c155ac8c0441f2b93d58a26f2097c1] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=36.6 K 2024-11-24T08:13:53,854 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24f38815ac774296a070f21ffc7ee0ea, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732436031307 2024-11-24T08:13:53,854 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 130f04a41a7344d6854825a8cf59fcf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732436032499 2024-11-24T08:13:53,855 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52c155ac8c0441f2b93d58a26f2097c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732436033651 2024-11-24T08:13:53,877 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:53,878 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/54093ccf86e44f5791fc6ac9e3a309c4 is 50, key is test_row_0/A:col10/1732436033651/Put/seqid=0 2024-11-24T08:13:53,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741896_1072 (size=12915) 2024-11-24T08:13:53,893 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/0573adb83d4540ff96ab92540999cc39 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/0573adb83d4540ff96ab92540999cc39 2024-11-24T08:13:53,902 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 187d8f78b18a17144230e52cfd716831/C of 187d8f78b18a17144230e52cfd716831 into 0573adb83d4540ff96ab92540999cc39(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:53,902 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:53,902 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/C, priority=12, startTime=1732436033336; duration=0sec 2024-11-24T08:13:53,902 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T08:13:53,902 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:C 2024-11-24T08:13:53,903 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:C 2024-11-24T08:13:53,903 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:53,904 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:53,904 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/B is initiating minor compaction (all files) 2024-11-24T08:13:53,904 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/B in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:53,904 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5eb43b2e54aa44cdbd8d2da7b55c35eb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/638087b37d414a86a95c38a4078da98d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d76f8e7055844870a3d50f317efe9306] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=36.6 K 2024-11-24T08:13:53,906 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eb43b2e54aa44cdbd8d2da7b55c35eb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732436031307 2024-11-24T08:13:53,907 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 638087b37d414a86a95c38a4078da98d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732436032499 2024-11-24T08:13:53,908 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting d76f8e7055844870a3d50f317efe9306, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732436033651 2024-11-24T08:13:53,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:53,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-24T08:13:53,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:53,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:13:53,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:53,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:13:53,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:53,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:13:53,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741897_1073 (size=12983) 2024-11-24T08:13:53,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/09005ac5f7454a52a32acb048a669fff is 50, key is test_row_0/A:col10/1732436033759/Put/seqid=0 2024-11-24T08:13:53,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T08:13:53,934 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#B#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:53,935 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/5f23de193a4d49038b92b2a0009f7acf is 50, key is test_row_0/B:col10/1732436033651/Put/seqid=0 2024-11-24T08:13:53,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436093951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436093954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:53,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741899_1075 (size=12983) 2024-11-24T08:13:53,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741898_1074 (size=12301) 2024-11-24T08:13:53,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T08:13:53,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:53,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:53,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:53,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:53,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:53,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:53,989 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/5f23de193a4d49038b92b2a0009f7acf as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5f23de193a4d49038b92b2a0009f7acf 2024-11-24T08:13:54,001 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/B of 187d8f78b18a17144230e52cfd716831 into 5f23de193a4d49038b92b2a0009f7acf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:54,001 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:54,001 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/B, priority=13, startTime=1732436033852; duration=0sec 2024-11-24T08:13:54,001 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:54,002 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:B 2024-11-24T08:13:54,002 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T08:13:54,003 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T08:13:54,004 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T08:13:54,005 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. because compaction request was cancelled 2024-11-24T08:13:54,005 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:C 2024-11-24T08:13:54,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436094030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436094031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,037 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436094060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436094060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436094070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T08:13:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:54,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T08:13:54,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436094270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436094280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T08:13:54,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:54,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:54,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:54,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,329 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/54093ccf86e44f5791fc6ac9e3a309c4 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/54093ccf86e44f5791fc6ac9e3a309c4 2024-11-24T08:13:54,336 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/A of 187d8f78b18a17144230e52cfd716831 into 54093ccf86e44f5791fc6ac9e3a309c4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:54,336 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:54,336 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/A, priority=13, startTime=1732436033851; duration=0sec 2024-11-24T08:13:54,337 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:54,337 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:A 2024-11-24T08:13:54,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436094340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,346 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436094340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436094370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/09005ac5f7454a52a32acb048a669fff 2024-11-24T08:13:54,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d756c2eacd754ded8d81089b1d549132 is 50, key is test_row_0/B:col10/1732436033759/Put/seqid=0 2024-11-24T08:13:54,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741900_1076 (size=12301) 2024-11-24T08:13:54,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T08:13:54,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:54,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:54,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:54,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T08:13:54,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436094580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436094589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T08:13:54,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:54,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:54,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:54,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T08:13:54,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:54,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:54,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:54,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d756c2eacd754ded8d81089b1d549132 2024-11-24T08:13:54,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/7b53a96d11664b2598176c7e1c16f70b is 50, key is test_row_0/C:col10/1732436033759/Put/seqid=0 2024-11-24T08:13:54,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741901_1077 (size=12301) 2024-11-24T08:13:54,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436094859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436094860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436094880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:54,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:54,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T08:13:54,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:54,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:54,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:54,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T08:13:55,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T08:13:55,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:55,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:55,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:55,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:55,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:55,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436095091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:55,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:55,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:55,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436095100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:55,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:55,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T08:13:55,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:55,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:55,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:55,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:55,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:55,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/7b53a96d11664b2598176c7e1c16f70b 2024-11-24T08:13:55,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/09005ac5f7454a52a32acb048a669fff as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/09005ac5f7454a52a32acb048a669fff 2024-11-24T08:13:55,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/09005ac5f7454a52a32acb048a669fff, entries=150, sequenceid=320, filesize=12.0 K 2024-11-24T08:13:55,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d756c2eacd754ded8d81089b1d549132 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d756c2eacd754ded8d81089b1d549132 2024-11-24T08:13:55,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d756c2eacd754ded8d81089b1d549132, entries=150, sequenceid=320, filesize=12.0 K 2024-11-24T08:13:55,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/7b53a96d11664b2598176c7e1c16f70b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/7b53a96d11664b2598176c7e1c16f70b 2024-11-24T08:13:55,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/7b53a96d11664b2598176c7e1c16f70b, entries=150, sequenceid=320, filesize=12.0 K 2024-11-24T08:13:55,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 187d8f78b18a17144230e52cfd716831 in 1355ms, sequenceid=320, compaction requested=true 2024-11-24T08:13:55,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:55,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:13:55,265 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T08:13:55,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:55,265 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T08:13:55,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:13:55,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:55,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:13:55,265 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:55,267 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T08:13:55,267 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T08:13:55,267 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. because compaction request was cancelled 2024-11-24T08:13:55,267 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:B 2024-11-24T08:13:55,267 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:55,268 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-24T08:13:55,268 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-24T08:13:55,268 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. because compaction request was cancelled 2024-11-24T08:13:55,268 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:A 2024-11-24T08:13:55,269 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:55,269 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/C is initiating minor compaction (all files) 2024-11-24T08:13:55,269 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/C in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:55,269 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/0573adb83d4540ff96ab92540999cc39, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/a04f64d617d242bdbcd281a4210f8f59, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/7b53a96d11664b2598176c7e1c16f70b] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=36.6 K 2024-11-24T08:13:55,270 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0573adb83d4540ff96ab92540999cc39, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732436032499 2024-11-24T08:13:55,270 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting a04f64d617d242bdbcd281a4210f8f59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732436033651 2024-11-24T08:13:55,271 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b53a96d11664b2598176c7e1c16f70b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732436033759 2024-11-24T08:13:55,288 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#C#compaction#65 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:55,288 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/245e077c18fc49b0be085a26368ff4bb is 50, key is test_row_0/C:col10/1732436033759/Put/seqid=0 2024-11-24T08:13:55,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741902_1078 (size=13017) 2024-11-24T08:13:55,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-24T08:13:55,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:55,374 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:13:55,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:55,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:55,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:55,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:55,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:55,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:55,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/408dae121add4c2bac500078a4459edd is 50, key is test_row_0/A:col10/1732436033939/Put/seqid=0 2024-11-24T08:13:55,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741903_1079 (size=12301) 2024-11-24T08:13:55,391 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/408dae121add4c2bac500078a4459edd 2024-11-24T08:13:55,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d0e6b3782e8e43578dad59b1e1dbacd2 is 50, key is test_row_0/B:col10/1732436033939/Put/seqid=0 2024-11-24T08:13:55,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741904_1080 (size=12301) 2024-11-24T08:13:55,722 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/245e077c18fc49b0be085a26368ff4bb as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/245e077c18fc49b0be085a26368ff4bb 2024-11-24T08:13:55,730 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/C of 187d8f78b18a17144230e52cfd716831 into 245e077c18fc49b0be085a26368ff4bb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:55,730 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:55,730 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/C, priority=13, startTime=1732436035265; duration=0sec 2024-11-24T08:13:55,731 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:55,731 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:C 2024-11-24T08:13:55,825 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d0e6b3782e8e43578dad59b1e1dbacd2 2024-11-24T08:13:55,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/69445ac6f9ea4e8fa8f253a19189c345 is 50, key is test_row_0/C:col10/1732436033939/Put/seqid=0 2024-11-24T08:13:55,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741905_1081 (size=12301) 2024-11-24T08:13:55,848 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/69445ac6f9ea4e8fa8f253a19189c345 2024-11-24T08:13:55,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/408dae121add4c2bac500078a4459edd as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/408dae121add4c2bac500078a4459edd 2024-11-24T08:13:55,865 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/408dae121add4c2bac500078a4459edd, entries=150, sequenceid=332, filesize=12.0 K 2024-11-24T08:13:55,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d0e6b3782e8e43578dad59b1e1dbacd2 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d0e6b3782e8e43578dad59b1e1dbacd2 2024-11-24T08:13:55,874 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d0e6b3782e8e43578dad59b1e1dbacd2, entries=150, sequenceid=332, filesize=12.0 K 2024-11-24T08:13:55,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/69445ac6f9ea4e8fa8f253a19189c345 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/69445ac6f9ea4e8fa8f253a19189c345 2024-11-24T08:13:55,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:55,892 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/69445ac6f9ea4e8fa8f253a19189c345, entries=150, sequenceid=332, filesize=12.0 K 2024-11-24T08:13:55,894 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=26.84 KB/27480 for 187d8f78b18a17144230e52cfd716831 in 520ms, sequenceid=332, compaction requested=true 2024-11-24T08:13:55,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:55,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:55,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-24T08:13:55,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-24T08:13:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:55,900 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:13:55,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:55,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:55,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:55,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:55,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:55,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:55,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-24T08:13:55,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0720 sec 2024-11-24T08:13:55,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/58c8d8320a1448019cac3b413e00ef7e is 50, key is test_row_0/A:col10/1732436035897/Put/seqid=0 2024-11-24T08:13:55,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.0860 sec 2024-11-24T08:13:55,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741906_1082 (size=17181) 2024-11-24T08:13:55,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/58c8d8320a1448019cac3b413e00ef7e 2024-11-24T08:13:55,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/380a725ba429466880b5439b0888b74a is 50, key is test_row_0/B:col10/1732436035897/Put/seqid=0 2024-11-24T08:13:55,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741907_1083 (size=12301) 2024-11-24T08:13:55,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436095956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:55,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436095963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:55,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436035945, max timestamp=1732436035960Num uniques 859; 2024-11-24T08:13:55,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436035945, max timestamp=1732436035960Num uniques 859; 2024-11-24T08:13:55,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:13:55,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:13:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:55,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436035945, max timestamp=1732436035960Num uniques 859; 2024-11-24T08:13:55,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:13:55,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-24T08:13:55,970 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:13:55,971 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:13:55,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-24T08:13:55,974 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:13:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T08:13:55,983 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:13:55,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:13:56,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436096070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:56,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436096071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:56,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:56,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436096073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:56,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,082 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:56,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T08:13:56,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436096100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:56,108 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_0' in 187d8f78b18a17144230e52cfd716831 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:13:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,109 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:56,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:56,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436096120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:56,126 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_0' in 187d8f78b18a17144230e52cfd716831 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:13:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:56,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T08:13:56,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:56,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:56,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:56,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436096190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T08:13:56,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T08:13:56,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:56,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:56,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:56,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436096289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:56,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436096290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:56,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/380a725ba429466880b5439b0888b74a 2024-11-24T08:13:56,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/891f7401ff3b4c0bacc4f686d99d82de is 50, key is test_row_0/C:col10/1732436035897/Put/seqid=0 2024-11-24T08:13:56,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741908_1084 (size=12301) 2024-11-24T08:13:56,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436096400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:56,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T08:13:56,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:56,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:56,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:56,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T08:13:56,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:56,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:56,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:56,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T08:13:56,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436096601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:56,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436096601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:56,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:56,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436096710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:56,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:56,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T08:13:56,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:56,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:56,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:56,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/891f7401ff3b4c0bacc4f686d99d82de 2024-11-24T08:13:56,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/58c8d8320a1448019cac3b413e00ef7e as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/58c8d8320a1448019cac3b413e00ef7e 2024-11-24T08:13:56,785 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/58c8d8320a1448019cac3b413e00ef7e, entries=250, sequenceid=344, filesize=16.8 K 2024-11-24T08:13:56,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/380a725ba429466880b5439b0888b74a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/380a725ba429466880b5439b0888b74a 2024-11-24T08:13:56,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/380a725ba429466880b5439b0888b74a, entries=150, sequenceid=344, filesize=12.0 K 2024-11-24T08:13:56,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/891f7401ff3b4c0bacc4f686d99d82de as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/891f7401ff3b4c0bacc4f686d99d82de 2024-11-24T08:13:56,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/891f7401ff3b4c0bacc4f686d99d82de, entries=150, sequenceid=344, filesize=12.0 K 2024-11-24T08:13:56,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 187d8f78b18a17144230e52cfd716831 in 900ms, sequenceid=344, compaction requested=true 2024-11-24T08:13:56,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:56,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:13:56,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:56,800 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:13:56,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:13:56,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:56,800 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:13:56,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:13:56,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:56,802 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54766 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:13:56,802 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/A is initiating minor compaction (all files) 2024-11-24T08:13:56,802 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/A in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:56,802 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/54093ccf86e44f5791fc6ac9e3a309c4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/09005ac5f7454a52a32acb048a669fff, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/408dae121add4c2bac500078a4459edd, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/58c8d8320a1448019cac3b413e00ef7e] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=53.5 K 2024-11-24T08:13:56,802 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:13:56,802 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/B is initiating minor compaction (all files) 2024-11-24T08:13:56,803 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/B in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:56,803 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5f23de193a4d49038b92b2a0009f7acf, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d756c2eacd754ded8d81089b1d549132, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d0e6b3782e8e43578dad59b1e1dbacd2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/380a725ba429466880b5439b0888b74a] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=48.7 K 2024-11-24T08:13:56,803 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54093ccf86e44f5791fc6ac9e3a309c4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732436033651 2024-11-24T08:13:56,803 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f23de193a4d49038b92b2a0009f7acf, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1732436033651 2024-11-24T08:13:56,804 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09005ac5f7454a52a32acb048a669fff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732436033759 2024-11-24T08:13:56,804 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting d756c2eacd754ded8d81089b1d549132, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732436033759 2024-11-24T08:13:56,804 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 408dae121add4c2bac500078a4459edd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732436033939 2024-11-24T08:13:56,805 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting d0e6b3782e8e43578dad59b1e1dbacd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732436033939 2024-11-24T08:13:56,805 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58c8d8320a1448019cac3b413e00ef7e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732436035888 2024-11-24T08:13:56,805 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 380a725ba429466880b5439b0888b74a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732436035895 2024-11-24T08:13:56,817 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#A#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:56,818 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/4677e4475470435983c86c6bf4d5d041 is 50, key is test_row_0/A:col10/1732436035897/Put/seqid=0 2024-11-24T08:13:56,818 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#B#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:56,819 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/9eab12d798b946079a758f5eb9009bc6 is 50, key is test_row_0/B:col10/1732436035897/Put/seqid=0 2024-11-24T08:13:56,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741910_1086 (size=13119) 2024-11-24T08:13:56,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741909_1085 (size=13119) 2024-11-24T08:13:56,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-24T08:13:56,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:56,909 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-24T08:13:56,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:56,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:13:56,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:56,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:13:56,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:56,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:13:56,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/f0c70c7c7de640069ed8d365956a1410 is 50, key is test_row_0/A:col10/1732436036068/Put/seqid=0 2024-11-24T08:13:56,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741911_1087 (size=12301) 2024-11-24T08:13:56,928 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/f0c70c7c7de640069ed8d365956a1410 2024-11-24T08:13:56,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/864c3fb03b8b42dd9751641f35a507e4 is 50, key is test_row_0/B:col10/1732436036068/Put/seqid=0 2024-11-24T08:13:56,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741912_1088 (size=12301) 2024-11-24T08:13:56,949 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/864c3fb03b8b42dd9751641f35a507e4 2024-11-24T08:13:56,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/c70d1278a0b345e09f89c0efe8f97409 is 50, key is test_row_0/C:col10/1732436036068/Put/seqid=0 2024-11-24T08:13:56,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741913_1089 (size=12301) 2024-11-24T08:13:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T08:13:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:57,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:57,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436097135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:57,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:57,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436097138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:57,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436097220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:57,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:57,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436097240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:57,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:57,250 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/9eab12d798b946079a758f5eb9009bc6 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/9eab12d798b946079a758f5eb9009bc6 2024-11-24T08:13:57,251 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/4677e4475470435983c86c6bf4d5d041 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/4677e4475470435983c86c6bf4d5d041 2024-11-24T08:13:57,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436097262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:57,271 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 187d8f78b18a17144230e52cfd716831/A of 187d8f78b18a17144230e52cfd716831 into 4677e4475470435983c86c6bf4d5d041(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:57,271 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 187d8f78b18a17144230e52cfd716831/B of 187d8f78b18a17144230e52cfd716831 into 9eab12d798b946079a758f5eb9009bc6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:57,271 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:57,272 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:57,272 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/A, priority=12, startTime=1732436036800; duration=0sec 2024-11-24T08:13:57,272 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/B, priority=12, startTime=1732436036800; duration=0sec 2024-11-24T08:13:57,272 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:57,272 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:A 2024-11-24T08:13:57,272 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:57,272 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:B 2024-11-24T08:13:57,272 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:57,275 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:57,275 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/C is initiating minor compaction (all files) 2024-11-24T08:13:57,275 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/C in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:57,275 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/245e077c18fc49b0be085a26368ff4bb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/69445ac6f9ea4e8fa8f253a19189c345, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/891f7401ff3b4c0bacc4f686d99d82de] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=36.7 K 2024-11-24T08:13:57,276 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 245e077c18fc49b0be085a26368ff4bb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1732436033759 2024-11-24T08:13:57,276 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69445ac6f9ea4e8fa8f253a19189c345, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732436033939 2024-11-24T08:13:57,277 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 891f7401ff3b4c0bacc4f686d99d82de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732436035895 2024-11-24T08:13:57,298 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#C#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:57,299 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/4200ac8cd06e400d9677fa4cac932646 is 50, key is test_row_0/C:col10/1732436035897/Put/seqid=0 2024-11-24T08:13:57,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741914_1090 (size=13119) 2024-11-24T08:13:57,338 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/4200ac8cd06e400d9677fa4cac932646 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/4200ac8cd06e400d9677fa4cac932646 2024-11-24T08:13:57,351 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/C of 187d8f78b18a17144230e52cfd716831 into 4200ac8cd06e400d9677fa4cac932646(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:57,351 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:57,352 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/C, priority=13, startTime=1732436036800; duration=0sec 2024-11-24T08:13:57,352 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:57,352 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:C 2024-11-24T08:13:57,382 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/c70d1278a0b345e09f89c0efe8f97409 2024-11-24T08:13:57,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/f0c70c7c7de640069ed8d365956a1410 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/f0c70c7c7de640069ed8d365956a1410 2024-11-24T08:13:57,401 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/f0c70c7c7de640069ed8d365956a1410, entries=150, sequenceid=380, filesize=12.0 K 2024-11-24T08:13:57,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/864c3fb03b8b42dd9751641f35a507e4 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/864c3fb03b8b42dd9751641f35a507e4 2024-11-24T08:13:57,412 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/864c3fb03b8b42dd9751641f35a507e4, entries=150, sequenceid=380, filesize=12.0 K 2024-11-24T08:13:57,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/c70d1278a0b345e09f89c0efe8f97409 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c70d1278a0b345e09f89c0efe8f97409 2024-11-24T08:13:57,420 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c70d1278a0b345e09f89c0efe8f97409, entries=150, sequenceid=380, filesize=12.0 K 2024-11-24T08:13:57,422 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 187d8f78b18a17144230e52cfd716831 in 512ms, sequenceid=380, compaction requested=false 2024-11-24T08:13:57,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:57,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:57,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-24T08:13:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-24T08:13:57,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-24T08:13:57,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4410 sec 2024-11-24T08:13:57,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.4570 sec 2024-11-24T08:13:57,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T08:13:57,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:57,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:57,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:57,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:57,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:57,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:57,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/64f0c6e8bda040e79c342d0b9867ed31 is 50, key is test_row_0/A:col10/1732436037131/Put/seqid=0 2024-11-24T08:13:57,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436097519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:57,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436097523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:57,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:57,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741915_1091 (size=9857) 2024-11-24T08:13:57,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436097630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:57,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:57,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436097630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:57,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:57,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436097841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:57,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436097841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:57,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:57,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/64f0c6e8bda040e79c342d0b9867ed31 2024-11-24T08:13:57,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/258bceeccb8447ee91a03c2ec99e3d48 is 50, key is test_row_0/B:col10/1732436037131/Put/seqid=0 2024-11-24T08:13:57,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741916_1092 (size=9857) 2024-11-24T08:13:58,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-24T08:13:58,121 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:13:58,122 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:13:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-24T08:13:58,125 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:13:58,127 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:13:58,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:13:58,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T08:13:58,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436098130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:58,133 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_0' in 187d8f78b18a17144230e52cfd716831 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:13:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:58,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436098150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:58,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436098150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:58,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436098150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:58,153 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_0' in 187d8f78b18a17144230e52cfd716831 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:13:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:58,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:58,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:58,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436098230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:58,233 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 187d8f78b18a17144230e52cfd716831 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:13:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:58,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T08:13:58,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T08:13:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:58,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:58,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/258bceeccb8447ee91a03c2ec99e3d48 2024-11-24T08:13:58,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/08e9460010fc46ecad115060e1abbe8c is 50, key is test_row_0/C:col10/1732436037131/Put/seqid=0 2024-11-24T08:13:58,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741917_1093 (size=9857) 2024-11-24T08:13:58,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T08:13:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:58,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:58,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T08:13:58,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T08:13:58,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:58,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:58,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:58,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:58,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:58,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:58,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:58,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436098660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:58,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:58,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436098670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:58,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:58,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T08:13:58,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:58,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:58,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:58,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:58,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T08:13:58,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/08e9460010fc46ecad115060e1abbe8c 2024-11-24T08:13:58,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/64f0c6e8bda040e79c342d0b9867ed31 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/64f0c6e8bda040e79c342d0b9867ed31 2024-11-24T08:13:58,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/64f0c6e8bda040e79c342d0b9867ed31, entries=100, sequenceid=395, filesize=9.6 K 2024-11-24T08:13:58,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/258bceeccb8447ee91a03c2ec99e3d48 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/258bceeccb8447ee91a03c2ec99e3d48 2024-11-24T08:13:58,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/258bceeccb8447ee91a03c2ec99e3d48, entries=100, sequenceid=395, filesize=9.6 K 2024-11-24T08:13:58,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/08e9460010fc46ecad115060e1abbe8c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/08e9460010fc46ecad115060e1abbe8c 2024-11-24T08:13:58,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/08e9460010fc46ecad115060e1abbe8c, entries=100, sequenceid=395, filesize=9.6 K 2024-11-24T08:13:58,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 187d8f78b18a17144230e52cfd716831 in 1377ms, sequenceid=395, compaction requested=true 2024-11-24T08:13:58,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:58,832 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:58,833 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:58,833 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/A is initiating minor compaction (all files) 2024-11-24T08:13:58,833 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/A in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:58,834 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/4677e4475470435983c86c6bf4d5d041, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/f0c70c7c7de640069ed8d365956a1410, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/64f0c6e8bda040e79c342d0b9867ed31] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=34.5 K 2024-11-24T08:13:58,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:13:58,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:58,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:13:58,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:58,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:13:58,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:58,834 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:58,834 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4677e4475470435983c86c6bf4d5d041, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732436035895 2024-11-24T08:13:58,835 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0c70c7c7de640069ed8d365956a1410, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732436036023 2024-11-24T08:13:58,835 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64f0c6e8bda040e79c342d0b9867ed31, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732436037131 2024-11-24T08:13:58,836 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:58,836 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/B is initiating minor compaction (all files) 2024-11-24T08:13:58,836 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/B in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:58,836 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/9eab12d798b946079a758f5eb9009bc6, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/864c3fb03b8b42dd9751641f35a507e4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/258bceeccb8447ee91a03c2ec99e3d48] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=34.5 K 2024-11-24T08:13:58,837 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eab12d798b946079a758f5eb9009bc6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732436035895 2024-11-24T08:13:58,838 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 864c3fb03b8b42dd9751641f35a507e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732436036023 2024-11-24T08:13:58,839 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 258bceeccb8447ee91a03c2ec99e3d48, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732436037131 2024-11-24T08:13:58,859 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#A#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:58,860 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/158235b2bd634e6bb93830e4719b3cb8 is 50, key is test_row_0/A:col10/1732436037131/Put/seqid=0 2024-11-24T08:13:58,866 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#B#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:58,867 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/660f07c8bbe54545a28e892a45f4d650 is 50, key is test_row_0/B:col10/1732436037131/Put/seqid=0 2024-11-24T08:13:58,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-24T08:13:58,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:58,898 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T08:13:58,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:58,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:58,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:58,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:58,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:58,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:58,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/f3c66699181f42c5b2a762cc447eab31 is 50, key is test_row_0/A:col10/1732436037493/Put/seqid=0 2024-11-24T08:13:58,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741919_1095 (size=13221) 2024-11-24T08:13:58,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741918_1094 (size=13221) 2024-11-24T08:13:58,919 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/660f07c8bbe54545a28e892a45f4d650 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/660f07c8bbe54545a28e892a45f4d650 2024-11-24T08:13:58,929 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/B of 187d8f78b18a17144230e52cfd716831 into 660f07c8bbe54545a28e892a45f4d650(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:58,929 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:58,929 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/B, priority=13, startTime=1732436038834; duration=0sec 2024-11-24T08:13:58,929 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:13:58,929 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:B 2024-11-24T08:13:58,929 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:13:58,932 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:13:58,933 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/C is initiating minor compaction (all files) 2024-11-24T08:13:58,933 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/C in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:58,933 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/4200ac8cd06e400d9677fa4cac932646, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c70d1278a0b345e09f89c0efe8f97409, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/08e9460010fc46ecad115060e1abbe8c] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=34.5 K 2024-11-24T08:13:58,933 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4200ac8cd06e400d9677fa4cac932646, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1732436035895 2024-11-24T08:13:58,934 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting c70d1278a0b345e09f89c0efe8f97409, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732436036023 2024-11-24T08:13:58,934 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 08e9460010fc46ecad115060e1abbe8c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732436037131 2024-11-24T08:13:58,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741920_1096 (size=12301) 2024-11-24T08:13:58,941 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/f3c66699181f42c5b2a762cc447eab31 2024-11-24T08:13:58,948 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#C#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:13:58,949 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/5f7265da7ec64561a442ecdbb358e367 is 50, key is test_row_0/C:col10/1732436037131/Put/seqid=0 2024-11-24T08:13:58,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d5c470da690e4805bcbf625cede32a7b is 50, key is test_row_0/B:col10/1732436037493/Put/seqid=0 2024-11-24T08:13:58,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741921_1097 (size=13221) 2024-11-24T08:13:58,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741922_1098 (size=12301) 2024-11-24T08:13:58,979 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d5c470da690e4805bcbf625cede32a7b 2024-11-24T08:13:58,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/400b8d08b792463b86e93ec5b27e1bbc is 50, key is test_row_0/C:col10/1732436037493/Put/seqid=0 2024-11-24T08:13:58,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741923_1099 (size=12301) 2024-11-24T08:13:59,000 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/400b8d08b792463b86e93ec5b27e1bbc 2024-11-24T08:13:59,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/f3c66699181f42c5b2a762cc447eab31 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/f3c66699181f42c5b2a762cc447eab31 2024-11-24T08:13:59,016 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/f3c66699181f42c5b2a762cc447eab31, entries=150, sequenceid=415, filesize=12.0 K 2024-11-24T08:13:59,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/d5c470da690e4805bcbf625cede32a7b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d5c470da690e4805bcbf625cede32a7b 2024-11-24T08:13:59,029 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d5c470da690e4805bcbf625cede32a7b, entries=150, sequenceid=415, filesize=12.0 K 2024-11-24T08:13:59,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/400b8d08b792463b86e93ec5b27e1bbc as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/400b8d08b792463b86e93ec5b27e1bbc 2024-11-24T08:13:59,039 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/400b8d08b792463b86e93ec5b27e1bbc, entries=150, sequenceid=415, filesize=12.0 K 2024-11-24T08:13:59,041 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 187d8f78b18a17144230e52cfd716831 in 144ms, sequenceid=415, compaction requested=false 2024-11-24T08:13:59,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:59,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:59,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-24T08:13:59,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-24T08:13:59,045 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-24T08:13:59,045 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 916 msec 2024-11-24T08:13:59,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 924 msec 2024-11-24T08:13:59,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-24T08:13:59,270 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:13:59,271 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:13:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-24T08:13:59,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T08:13:59,273 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:13:59,274 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:13:59,275 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:13:59,323 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/158235b2bd634e6bb93830e4719b3cb8 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/158235b2bd634e6bb93830e4719b3cb8 2024-11-24T08:13:59,330 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/A of 187d8f78b18a17144230e52cfd716831 into 158235b2bd634e6bb93830e4719b3cb8(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:59,330 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:59,330 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/A, priority=13, startTime=1732436038832; duration=0sec 2024-11-24T08:13:59,331 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:59,331 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:A 2024-11-24T08:13:59,373 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/5f7265da7ec64561a442ecdbb358e367 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/5f7265da7ec64561a442ecdbb358e367 2024-11-24T08:13:59,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T08:13:59,382 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/C of 187d8f78b18a17144230e52cfd716831 into 5f7265da7ec64561a442ecdbb358e367(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:13:59,382 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:59,382 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/C, priority=13, startTime=1732436038834; duration=0sec 2024-11-24T08:13:59,382 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:13:59,382 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:C 2024-11-24T08:13:59,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-24T08:13:59,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:59,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:13:59,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:59,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-24T08:13:59,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-24T08:13:59,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-24T08:13:59,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-24T08:13:59,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 163 msec 2024-11-24T08:13:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-24T08:13:59,590 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:13:59,591 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:13:59,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-24T08:13:59,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T08:13:59,596 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:13:59,597 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:13:59,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:13:59,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T08:13:59,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:13:59,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:13:59,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:59,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:13:59,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:59,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:13:59,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:13:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:13:59,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/80345c688fb1423981f8cee307048f40 is 50, key is test_row_0/A:col10/1732436039700/Put/seqid=0 2024-11-24T08:13:59,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T08:13:59,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:59,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:59,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:59,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:59,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:59,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:59,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741924_1100 (size=17181) 2024-11-24T08:13:59,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/80345c688fb1423981f8cee307048f40 2024-11-24T08:13:59,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732436039757, max timestamp=1732436039778Num uniques 859; 2024-11-24T08:13:59,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:13:59,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732436039757, max timestamp=1732436039778Num uniques 859; 2024-11-24T08:13:59,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:13:59,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732436039757, max timestamp=1732436039778Num uniques 859; 2024-11-24T08:13:59,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:13:59,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/4b4593c1f84a48c388e14007a4c30ec3 is 50, key is test_row_0/B:col10/1732436039700/Put/seqid=0 2024-11-24T08:13:59,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741925_1101 (size=12301) 2024-11-24T08:13:59,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436099844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:59,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:59,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436099848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:59,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:59,857 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:59,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T08:13:59,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T08:13:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:13:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:13:59,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:59,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:13:59,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:59,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436099960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:59,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:13:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436099960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:13:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:13:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:13:59,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:00,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T08:14:00,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:00,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:14:00,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:00,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:00,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:00,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:00,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436100170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:00,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436100172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:00,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/4b4593c1f84a48c388e14007a4c30ec3 2024-11-24T08:14:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T08:14:00,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/fe25fe424b984089a32ef41b08c1ea67 is 50, key is test_row_0/C:col10/1732436039700/Put/seqid=0 2024-11-24T08:14:00,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T08:14:00,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:00,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:14:00,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:00,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:00,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:00,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:00,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741926_1102 (size=12301) 2024-11-24T08:14:00,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/fe25fe424b984089a32ef41b08c1ea67 2024-11-24T08:14:00,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/80345c688fb1423981f8cee307048f40 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/80345c688fb1423981f8cee307048f40 2024-11-24T08:14:00,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/80345c688fb1423981f8cee307048f40, entries=250, sequenceid=429, filesize=16.8 K 2024-11-24T08:14:00,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46190 deadline: 1732436100240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:00,242 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 187d8f78b18a17144230e52cfd716831 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:14:00,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:00,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/4b4593c1f84a48c388e14007a4c30ec3 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/4b4593c1f84a48c388e14007a4c30ec3 2024-11-24T08:14:00,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/4b4593c1f84a48c388e14007a4c30ec3, entries=150, sequenceid=429, filesize=12.0 K 2024-11-24T08:14:00,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/fe25fe424b984089a32ef41b08c1ea67 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/fe25fe424b984089a32ef41b08c1ea67 2024-11-24T08:14:00,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/fe25fe424b984089a32ef41b08c1ea67, entries=150, sequenceid=429, filesize=12.0 K 2024-11-24T08:14:00,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 187d8f78b18a17144230e52cfd716831 in 555ms, sequenceid=429, compaction requested=true 2024-11-24T08:14:00,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:14:00,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:00,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:00,257 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:00,257 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:00,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:00,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:00,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:00,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:00,259 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:00,259 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/A is initiating minor compaction (all files) 2024-11-24T08:14:00,259 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/A in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:00,259 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/158235b2bd634e6bb93830e4719b3cb8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/f3c66699181f42c5b2a762cc447eab31, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/80345c688fb1423981f8cee307048f40] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=41.7 K 2024-11-24T08:14:00,260 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:00,260 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 158235b2bd634e6bb93830e4719b3cb8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732436036058 2024-11-24T08:14:00,260 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/B is initiating minor compaction (all files) 2024-11-24T08:14:00,260 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/B in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:00,260 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/660f07c8bbe54545a28e892a45f4d650, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d5c470da690e4805bcbf625cede32a7b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/4b4593c1f84a48c388e14007a4c30ec3] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=36.9 K 2024-11-24T08:14:00,263 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3c66699181f42c5b2a762cc447eab31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732436037493 2024-11-24T08:14:00,263 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 660f07c8bbe54545a28e892a45f4d650, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732436036058 2024-11-24T08:14:00,263 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80345c688fb1423981f8cee307048f40, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732436039690 2024-11-24T08:14:00,264 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting d5c470da690e4805bcbf625cede32a7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732436037493 2024-11-24T08:14:00,265 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b4593c1f84a48c388e14007a4c30ec3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732436039696 2024-11-24T08:14:00,274 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#A#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:00,275 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/c1785f71727c4da1a6c3a5389f70d6ad is 50, key is test_row_0/A:col10/1732436039700/Put/seqid=0 2024-11-24T08:14:00,290 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:00,291 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/9bc15faa63414091bd8f19e75baba4e3 is 50, key is test_row_0/B:col10/1732436039700/Put/seqid=0 2024-11-24T08:14:00,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741927_1103 (size=13323) 2024-11-24T08:14:00,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741928_1104 (size=13323) 2024-11-24T08:14:00,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-24T08:14:00,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:00,377 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-24T08:14:00,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:14:00,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:00,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:14:00,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:00,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:14:00,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:00,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/db5e5447fbea49ab92035f8927f90f5c is 50, key is test_row_0/A:col10/1732436039846/Put/seqid=0 2024-11-24T08:14:00,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741929_1105 (size=12301) 2024-11-24T08:14:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:14:00,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:14:00,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436100515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:00,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436100526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:00,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436100629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:00,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436100640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:00,717 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/c1785f71727c4da1a6c3a5389f70d6ad as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/c1785f71727c4da1a6c3a5389f70d6ad 2024-11-24T08:14:00,726 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/A of 187d8f78b18a17144230e52cfd716831 into c1785f71727c4da1a6c3a5389f70d6ad(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:00,727 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:14:00,727 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/9bc15faa63414091bd8f19e75baba4e3 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/9bc15faa63414091bd8f19e75baba4e3 2024-11-24T08:14:00,727 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/A, priority=13, startTime=1732436040257; duration=0sec 2024-11-24T08:14:00,727 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:00,727 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:A 2024-11-24T08:14:00,727 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T08:14:00,730 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:00,731 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/C is initiating minor compaction (all files) 2024-11-24T08:14:00,731 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/C in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:00,731 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/5f7265da7ec64561a442ecdbb358e367, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/400b8d08b792463b86e93ec5b27e1bbc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/fe25fe424b984089a32ef41b08c1ea67] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=36.9 K 2024-11-24T08:14:00,732 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f7265da7ec64561a442ecdbb358e367, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732436036058 2024-11-24T08:14:00,732 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 400b8d08b792463b86e93ec5b27e1bbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1732436037493 2024-11-24T08:14:00,733 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe25fe424b984089a32ef41b08c1ea67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732436039696 2024-11-24T08:14:00,734 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/B of 187d8f78b18a17144230e52cfd716831 into 9bc15faa63414091bd8f19e75baba4e3(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:00,734 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:14:00,734 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/B, priority=13, startTime=1732436040257; duration=0sec 2024-11-24T08:14:00,734 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:00,734 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:B 2024-11-24T08:14:00,742 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#C#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:00,743 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/faf19e8eef0342439954d78eef93b4f5 is 50, key is test_row_0/C:col10/1732436039700/Put/seqid=0 2024-11-24T08:14:00,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741930_1106 (size=13323) 2024-11-24T08:14:00,758 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/faf19e8eef0342439954d78eef93b4f5 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/faf19e8eef0342439954d78eef93b4f5 2024-11-24T08:14:00,764 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/C of 187d8f78b18a17144230e52cfd716831 into faf19e8eef0342439954d78eef93b4f5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:00,764 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:14:00,764 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/C, priority=13, startTime=1732436040258; duration=0sec 2024-11-24T08:14:00,765 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:00,765 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:C 2024-11-24T08:14:00,793 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/db5e5447fbea49ab92035f8927f90f5c 2024-11-24T08:14:00,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/080e8463f5084cb0889d1a90999337e5 is 50, key is test_row_0/B:col10/1732436039846/Put/seqid=0 2024-11-24T08:14:00,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741931_1107 (size=12301) 2024-11-24T08:14:00,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436100839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:00,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:00,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436100850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:00,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:01,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436101150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:01,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436101159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:01,208 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/080e8463f5084cb0889d1a90999337e5 2024-11-24T08:14:01,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/40c3abbf00d149f9b94e0d032ed9ca70 is 50, key is test_row_0/C:col10/1732436039846/Put/seqid=0 2024-11-24T08:14:01,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741932_1108 (size=12301) 2024-11-24T08:14:01,623 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/40c3abbf00d149f9b94e0d032ed9ca70 2024-11-24T08:14:01,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/db5e5447fbea49ab92035f8927f90f5c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/db5e5447fbea49ab92035f8927f90f5c 2024-11-24T08:14:01,636 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/db5e5447fbea49ab92035f8927f90f5c, entries=150, sequenceid=465, filesize=12.0 K 2024-11-24T08:14:01,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/080e8463f5084cb0889d1a90999337e5 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/080e8463f5084cb0889d1a90999337e5 2024-11-24T08:14:01,643 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/080e8463f5084cb0889d1a90999337e5, entries=150, sequenceid=465, filesize=12.0 K 2024-11-24T08:14:01,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/40c3abbf00d149f9b94e0d032ed9ca70 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/40c3abbf00d149f9b94e0d032ed9ca70 2024-11-24T08:14:01,651 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/40c3abbf00d149f9b94e0d032ed9ca70, entries=150, sequenceid=465, filesize=12.0 K 2024-11-24T08:14:01,652 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 187d8f78b18a17144230e52cfd716831 in 1275ms, sequenceid=465, compaction requested=false 2024-11-24T08:14:01,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:14:01,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:01,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-24T08:14:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-24T08:14:01,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-24T08:14:01,656 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0570 sec 2024-11-24T08:14:01,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.0660 sec 2024-11-24T08:14:01,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 187d8f78b18a17144230e52cfd716831 2024-11-24T08:14:01,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T08:14:01,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:14:01,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:01,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:14:01,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:01,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:14:01,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:01,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/b848908a26cb4be1a307c6e10fc69260 is 50, key is test_row_0/A:col10/1732436040505/Put/seqid=0 2024-11-24T08:14:01,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741933_1109 (size=17181) 2024-11-24T08:14:01,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436101734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:01,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:01,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:01,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:01,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436101740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:01,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:01,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:01,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:01,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-24T08:14:01,750 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:01,750 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:01,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-24T08:14:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T08:14:01,752 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:01,753 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:01,753 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:01,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436101849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:01,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436101849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:01,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T08:14:01,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T08:14:01,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:01,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:14:01,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:01,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:01,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T08:14:02,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:14:02,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:02,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436102059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436102059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:02,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T08:14:02,071 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-24T08:14:02,071 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:02,075 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-24T08:14:02,075 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:02,076 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:02,076 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:02,076 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:02,076 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:02,076 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:02,076 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:02,079 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-24T08:14:02,079 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:02,079 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:02,079 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:02,079 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:02,080 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-24T08:14:02,080 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:02,080 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:02,080 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:02,080 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:02,086 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/b848908a26cb4be1a307c6e10fc69260 2024-11-24T08:14:02,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/b58b84988c5549bd937059b6a866ce42 is 50, key is test_row_0/B:col10/1732436040505/Put/seqid=0 2024-11-24T08:14:02,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741934_1110 (size=12301) 2024-11-24T08:14:02,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46272 deadline: 1732436102149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:02,150 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_0' in 187d8f78b18a17144230e52cfd716831 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:14:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,151 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:02,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46244 deadline: 1732436102200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:02,201 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_0' in 187d8f78b18a17144230e52cfd716831 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:14:02,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:02,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T08:14:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:14:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T08:14:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:14:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:02,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436102370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436102370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:02,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:02,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T08:14:02,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/b58b84988c5549bd937059b6a866ce42 2024-11-24T08:14:02,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/c1057df822ab434d973bae25e6eeea7c is 50, key is test_row_0/C:col10/1732436040505/Put/seqid=0 2024-11-24T08:14:02,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T08:14:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:14:02,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741935_1111 (size=12301) 2024-11-24T08:14:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T08:14:02,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:14:02,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T08:14:02,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. as already flushing 2024-11-24T08:14:02,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:02,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:02,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46226 deadline: 1732436102880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:02,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46200 deadline: 1732436102880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:02,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T08:14:02,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/c1057df822ab434d973bae25e6eeea7c 2024-11-24T08:14:02,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/b848908a26cb4be1a307c6e10fc69260 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/b848908a26cb4be1a307c6e10fc69260 2024-11-24T08:14:02,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/b848908a26cb4be1a307c6e10fc69260, entries=250, sequenceid=480, filesize=16.8 K 2024-11-24T08:14:02,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/b58b84988c5549bd937059b6a866ce42 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/b58b84988c5549bd937059b6a866ce42 2024-11-24T08:14:02,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/b58b84988c5549bd937059b6a866ce42, entries=150, sequenceid=480, filesize=12.0 K 2024-11-24T08:14:02,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/c1057df822ab434d973bae25e6eeea7c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c1057df822ab434d973bae25e6eeea7c 2024-11-24T08:14:02,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c1057df822ab434d973bae25e6eeea7c, entries=150, sequenceid=480, filesize=12.0 K 2024-11-24T08:14:02,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 187d8f78b18a17144230e52cfd716831 in 1276ms, sequenceid=480, compaction requested=true 2024-11-24T08:14:02,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:14:02,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:02,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:02,949 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:02,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:02,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:02,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 187d8f78b18a17144230e52cfd716831:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:02,949 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:02,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:02,950 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:02,950 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:02,950 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/A is initiating minor compaction (all files) 2024-11-24T08:14:02,950 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/B is initiating minor compaction (all files) 2024-11-24T08:14:02,950 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/A in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,951 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/B in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,951 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/c1785f71727c4da1a6c3a5389f70d6ad, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/db5e5447fbea49ab92035f8927f90f5c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/b848908a26cb4be1a307c6e10fc69260] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=41.8 K 2024-11-24T08:14:02,951 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/9bc15faa63414091bd8f19e75baba4e3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/080e8463f5084cb0889d1a90999337e5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/b58b84988c5549bd937059b6a866ce42] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=37.0 K 2024-11-24T08:14:02,951 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1785f71727c4da1a6c3a5389f70d6ad, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732436039696 2024-11-24T08:14:02,951 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bc15faa63414091bd8f19e75baba4e3, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732436039696 2024-11-24T08:14:02,951 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 080e8463f5084cb0889d1a90999337e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732436039827 2024-11-24T08:14:02,951 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting db5e5447fbea49ab92035f8927f90f5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732436039827 2024-11-24T08:14:02,952 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting b58b84988c5549bd937059b6a866ce42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732436040505 2024-11-24T08:14:02,952 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b848908a26cb4be1a307c6e10fc69260, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732436040496 2024-11-24T08:14:02,962 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#B#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:02,963 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/dd2ee642bb134b95a656636f0f8f6969 is 50, key is test_row_0/B:col10/1732436040505/Put/seqid=0 2024-11-24T08:14:02,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-24T08:14:02,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:02,981 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T08:14:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:14:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:14:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:14:02,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:02,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741936_1112 (size=13425) 2024-11-24T08:14:02,986 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#A#compaction#100 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:02,987 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/27ea5aaca22a46de93aa0e4424037612 is 50, key is test_row_0/A:col10/1732436040505/Put/seqid=0 2024-11-24T08:14:02,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/6cbda3228dce4cd0b28243d4347f09c0 is 50, key is test_row_0/A:col10/1732436041688/Put/seqid=0 2024-11-24T08:14:02,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741937_1113 (size=13425) 2024-11-24T08:14:03,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741938_1114 (size=12301) 2024-11-24T08:14:03,012 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/6cbda3228dce4cd0b28243d4347f09c0 2024-11-24T08:14:03,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/5d64c4f708ce438bb46163cf4c313480 is 50, key is test_row_0/B:col10/1732436041688/Put/seqid=0 2024-11-24T08:14:03,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741939_1115 (size=12301) 2024-11-24T08:14:03,030 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/5d64c4f708ce438bb46163cf4c313480 2024-11-24T08:14:03,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/c3f51e1185144cf9a06ac55a5812ca1c is 50, key is test_row_0/C:col10/1732436041688/Put/seqid=0 2024-11-24T08:14:03,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741940_1116 (size=12301) 2024-11-24T08:14:03,390 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/dd2ee642bb134b95a656636f0f8f6969 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/dd2ee642bb134b95a656636f0f8f6969 2024-11-24T08:14:03,395 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/B of 187d8f78b18a17144230e52cfd716831 into dd2ee642bb134b95a656636f0f8f6969(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:03,395 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:14:03,395 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/B, priority=13, startTime=1732436042949; duration=0sec 2024-11-24T08:14:03,395 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:03,395 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:B 2024-11-24T08:14:03,396 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:03,396 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/27ea5aaca22a46de93aa0e4424037612 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/27ea5aaca22a46de93aa0e4424037612 2024-11-24T08:14:03,396 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:03,397 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 187d8f78b18a17144230e52cfd716831/C is initiating minor compaction (all files) 2024-11-24T08:14:03,397 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 187d8f78b18a17144230e52cfd716831/C in TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:03,397 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/faf19e8eef0342439954d78eef93b4f5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/40c3abbf00d149f9b94e0d032ed9ca70, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c1057df822ab434d973bae25e6eeea7c] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp, totalSize=37.0 K 2024-11-24T08:14:03,397 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting faf19e8eef0342439954d78eef93b4f5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732436039696 2024-11-24T08:14:03,398 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 40c3abbf00d149f9b94e0d032ed9ca70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732436039827 2024-11-24T08:14:03,398 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting c1057df822ab434d973bae25e6eeea7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1732436040505 2024-11-24T08:14:03,401 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/A of 187d8f78b18a17144230e52cfd716831 into 27ea5aaca22a46de93aa0e4424037612(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:03,401 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:14:03,401 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/A, priority=13, startTime=1732436042949; duration=0sec 2024-11-24T08:14:03,402 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:03,402 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:A 2024-11-24T08:14:03,407 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 187d8f78b18a17144230e52cfd716831#C#compaction#104 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:03,407 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/01abef1cdd0344939a1a4a496ffe7c0b is 50, key is test_row_0/C:col10/1732436040505/Put/seqid=0 2024-11-24T08:14:03,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741941_1117 (size=13425) 2024-11-24T08:14:03,448 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=500 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/c3f51e1185144cf9a06ac55a5812ca1c 2024-11-24T08:14:03,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/6cbda3228dce4cd0b28243d4347f09c0 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/6cbda3228dce4cd0b28243d4347f09c0 2024-11-24T08:14:03,458 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/6cbda3228dce4cd0b28243d4347f09c0, entries=150, sequenceid=500, filesize=12.0 K 2024-11-24T08:14:03,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/5d64c4f708ce438bb46163cf4c313480 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5d64c4f708ce438bb46163cf4c313480 2024-11-24T08:14:03,463 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5d64c4f708ce438bb46163cf4c313480, entries=150, sequenceid=500, filesize=12.0 K 2024-11-24T08:14:03,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/c3f51e1185144cf9a06ac55a5812ca1c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c3f51e1185144cf9a06ac55a5812ca1c 2024-11-24T08:14:03,469 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c3f51e1185144cf9a06ac55a5812ca1c, entries=150, sequenceid=500, filesize=12.0 K 2024-11-24T08:14:03,470 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 187d8f78b18a17144230e52cfd716831 in 488ms, sequenceid=500, compaction requested=false 2024-11-24T08:14:03,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:14:03,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:03,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-24T08:14:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-24T08:14:03,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-24T08:14:03,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7180 sec 2024-11-24T08:14:03,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.7230 sec 2024-11-24T08:14:03,817 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/01abef1cdd0344939a1a4a496ffe7c0b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/01abef1cdd0344939a1a4a496ffe7c0b 2024-11-24T08:14:03,823 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 187d8f78b18a17144230e52cfd716831/C of 187d8f78b18a17144230e52cfd716831 into 01abef1cdd0344939a1a4a496ffe7c0b(size=13.1 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:03,823 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 187d8f78b18a17144230e52cfd716831: 2024-11-24T08:14:03,823 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., storeName=187d8f78b18a17144230e52cfd716831/C, priority=13, startTime=1732436042949; duration=0sec 2024-11-24T08:14:03,823 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:03,823 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 187d8f78b18a17144230e52cfd716831:C 2024-11-24T08:14:03,892 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-24T08:14:03,892 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:03,892 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:03,892 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:03,892 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-24T08:14:03,901 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:03,902 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-24T08:14:03,902 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:03,902 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:03,902 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:03,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:04,292 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-24T08:14:04,292 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:04,292 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:04,293 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:04,293 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:04,797 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T08:14:12,172 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-24T08:14:12,172 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:12,172 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:12,172 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:12,172 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:12,271 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-24T08:14:12,271 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:12,272 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:12,272 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:12,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:12,272 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T08:14:12,272 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-24T08:14:12,272 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-24T08:14:12,272 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-24T08:14:12,272 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-24T08:14:12,272 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-24T08:14:12,272 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T08:14:12,272 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2707 2024-11-24T08:14:12,272 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2746 2024-11-24T08:14:12,272 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T08:14:12,272 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1219 2024-11-24T08:14:12,272 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3641 rows 2024-11-24T08:14:12,272 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1221 2024-11-24T08:14:12,272 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3662 rows 2024-11-24T08:14:12,273 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T08:14:12,273 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:14:12,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:12,273 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:12,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:12,277 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T08:14:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:12,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T08:14:12,284 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436052284"}]},"ts":"1732436052284"} 2024-11-24T08:14:12,286 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T08:14:12,286 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T08:14:12,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T08:14:12,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=187d8f78b18a17144230e52cfd716831, UNASSIGN}] 2024-11-24T08:14:12,292 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=187d8f78b18a17144230e52cfd716831, UNASSIGN 2024-11-24T08:14:12,294 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=187d8f78b18a17144230e52cfd716831, regionState=CLOSING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:14:12,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=187d8f78b18a17144230e52cfd716831, UNASSIGN because future has completed 2024-11-24T08:14:12,296 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T08:14:12,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:14:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T08:14:12,452 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 187d8f78b18a17144230e52cfd716831 2024-11-24T08:14:12,452 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T08:14:12,453 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 187d8f78b18a17144230e52cfd716831, disabling compactions & flushes 2024-11-24T08:14:12,453 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:12,453 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:12,453 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. after waiting 0 ms 2024-11-24T08:14:12,453 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:12,453 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 187d8f78b18a17144230e52cfd716831 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T08:14:12,454 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=A 2024-11-24T08:14:12,454 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:12,454 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=B 2024-11-24T08:14:12,454 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:12,454 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 187d8f78b18a17144230e52cfd716831, store=C 2024-11-24T08:14:12,454 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:12,458 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/91912fba56964a259d29f7f979a4d57f is 50, key is test_row_0/A:col10/1732436052270/Put/seqid=0 2024-11-24T08:14:12,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741942_1118 (size=9857) 2024-11-24T08:14:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T08:14:12,862 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/91912fba56964a259d29f7f979a4d57f 2024-11-24T08:14:12,870 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/92da1b89c8bd45f89091cee791769bde is 50, key is test_row_0/B:col10/1732436052270/Put/seqid=0 2024-11-24T08:14:12,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741943_1119 (size=9857) 2024-11-24T08:14:12,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T08:14:13,274 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/92da1b89c8bd45f89091cee791769bde 2024-11-24T08:14:13,282 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/589bdd7b69d941cabb7f216f1a8b4c58 is 50, key is test_row_0/C:col10/1732436052270/Put/seqid=0 2024-11-24T08:14:13,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741944_1120 (size=9857) 2024-11-24T08:14:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T08:14:13,687 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/589bdd7b69d941cabb7f216f1a8b4c58 2024-11-24T08:14:13,692 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/A/91912fba56964a259d29f7f979a4d57f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/91912fba56964a259d29f7f979a4d57f 2024-11-24T08:14:13,697 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/91912fba56964a259d29f7f979a4d57f, entries=100, sequenceid=511, filesize=9.6 K 2024-11-24T08:14:13,697 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/B/92da1b89c8bd45f89091cee791769bde as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/92da1b89c8bd45f89091cee791769bde 2024-11-24T08:14:13,701 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/92da1b89c8bd45f89091cee791769bde, entries=100, sequenceid=511, filesize=9.6 K 2024-11-24T08:14:13,702 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/.tmp/C/589bdd7b69d941cabb7f216f1a8b4c58 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/589bdd7b69d941cabb7f216f1a8b4c58 2024-11-24T08:14:13,706 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/589bdd7b69d941cabb7f216f1a8b4c58, entries=100, sequenceid=511, filesize=9.6 K 2024-11-24T08:14:13,707 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 187d8f78b18a17144230e52cfd716831 in 1254ms, sequenceid=511, compaction requested=true 2024-11-24T08:14:13,708 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/523bef2d65a24f40a97c00547145caad, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/c9ac77af26514e0a845fdb38abd37deb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/eb8543c9c0e74e36a9f70b3bd3211e8f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/7bba3f577e7c48e1ac055e4c390b08ed, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/ccc32d4b2d6c4eaca4f7f7793d2d279c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/5b2e7a5326384794b91ed863ba671f26, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/85389a0a5c88400d967b4e394a12f0d1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/cf6faadb4d654bc3a6c824719adaf225, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/8a5ad919770c46018e2aac499190ee1a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/5c17b42c6b834a33be8e98f71223a54b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/bee96a9393d34b3b8d50c50762023d6f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/806c87ece1d04c029a60e5ff2554b6e4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/6dd4866554f4490caff85f41a6602d5b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/aa629b9b67484096946fc0c49d4bfa40, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/43fc05016c5a4a9a80700e9740eb2433, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/8566c2f8de7c44b7b89ca55b49ad0b87, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/24f38815ac774296a070f21ffc7ee0ea, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/ae3edc50af0444d78801254df5d94adf, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/130f04a41a7344d6854825a8cf59fcf9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/54093ccf86e44f5791fc6ac9e3a309c4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/52c155ac8c0441f2b93d58a26f2097c1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/09005ac5f7454a52a32acb048a669fff, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/408dae121add4c2bac500078a4459edd, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/58c8d8320a1448019cac3b413e00ef7e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/4677e4475470435983c86c6bf4d5d041, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/f0c70c7c7de640069ed8d365956a1410, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/158235b2bd634e6bb93830e4719b3cb8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/64f0c6e8bda040e79c342d0b9867ed31, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/f3c66699181f42c5b2a762cc447eab31, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/80345c688fb1423981f8cee307048f40, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/c1785f71727c4da1a6c3a5389f70d6ad, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/db5e5447fbea49ab92035f8927f90f5c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/b848908a26cb4be1a307c6e10fc69260] to archive 2024-11-24T08:14:13,711 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:14:13,716 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/523bef2d65a24f40a97c00547145caad to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/523bef2d65a24f40a97c00547145caad 2024-11-24T08:14:13,717 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/c9ac77af26514e0a845fdb38abd37deb to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/c9ac77af26514e0a845fdb38abd37deb 2024-11-24T08:14:13,718 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/eb8543c9c0e74e36a9f70b3bd3211e8f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/eb8543c9c0e74e36a9f70b3bd3211e8f 2024-11-24T08:14:13,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/7bba3f577e7c48e1ac055e4c390b08ed to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/7bba3f577e7c48e1ac055e4c390b08ed 2024-11-24T08:14:13,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/ccc32d4b2d6c4eaca4f7f7793d2d279c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/ccc32d4b2d6c4eaca4f7f7793d2d279c 2024-11-24T08:14:13,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/5b2e7a5326384794b91ed863ba671f26 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/5b2e7a5326384794b91ed863ba671f26 2024-11-24T08:14:13,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/85389a0a5c88400d967b4e394a12f0d1 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/85389a0a5c88400d967b4e394a12f0d1 2024-11-24T08:14:13,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/cf6faadb4d654bc3a6c824719adaf225 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/cf6faadb4d654bc3a6c824719adaf225 2024-11-24T08:14:13,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/8a5ad919770c46018e2aac499190ee1a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/8a5ad919770c46018e2aac499190ee1a 2024-11-24T08:14:13,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/5c17b42c6b834a33be8e98f71223a54b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/5c17b42c6b834a33be8e98f71223a54b 2024-11-24T08:14:13,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/bee96a9393d34b3b8d50c50762023d6f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/bee96a9393d34b3b8d50c50762023d6f 2024-11-24T08:14:13,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/806c87ece1d04c029a60e5ff2554b6e4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/806c87ece1d04c029a60e5ff2554b6e4 2024-11-24T08:14:13,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/6dd4866554f4490caff85f41a6602d5b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/6dd4866554f4490caff85f41a6602d5b 2024-11-24T08:14:13,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/aa629b9b67484096946fc0c49d4bfa40 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/aa629b9b67484096946fc0c49d4bfa40 2024-11-24T08:14:13,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/43fc05016c5a4a9a80700e9740eb2433 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/43fc05016c5a4a9a80700e9740eb2433 2024-11-24T08:14:13,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/8566c2f8de7c44b7b89ca55b49ad0b87 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/8566c2f8de7c44b7b89ca55b49ad0b87 2024-11-24T08:14:13,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/24f38815ac774296a070f21ffc7ee0ea to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/24f38815ac774296a070f21ffc7ee0ea 2024-11-24T08:14:13,736 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/ae3edc50af0444d78801254df5d94adf to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/ae3edc50af0444d78801254df5d94adf 2024-11-24T08:14:13,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/130f04a41a7344d6854825a8cf59fcf9 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/130f04a41a7344d6854825a8cf59fcf9 2024-11-24T08:14:13,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/54093ccf86e44f5791fc6ac9e3a309c4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/54093ccf86e44f5791fc6ac9e3a309c4 2024-11-24T08:14:13,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/52c155ac8c0441f2b93d58a26f2097c1 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/52c155ac8c0441f2b93d58a26f2097c1 2024-11-24T08:14:13,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/09005ac5f7454a52a32acb048a669fff to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/09005ac5f7454a52a32acb048a669fff 2024-11-24T08:14:13,742 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/408dae121add4c2bac500078a4459edd to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/408dae121add4c2bac500078a4459edd 2024-11-24T08:14:13,743 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/58c8d8320a1448019cac3b413e00ef7e to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/58c8d8320a1448019cac3b413e00ef7e 2024-11-24T08:14:13,745 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/4677e4475470435983c86c6bf4d5d041 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/4677e4475470435983c86c6bf4d5d041 2024-11-24T08:14:13,746 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/f0c70c7c7de640069ed8d365956a1410 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/f0c70c7c7de640069ed8d365956a1410 2024-11-24T08:14:13,747 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/158235b2bd634e6bb93830e4719b3cb8 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/158235b2bd634e6bb93830e4719b3cb8 2024-11-24T08:14:13,748 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/64f0c6e8bda040e79c342d0b9867ed31 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/64f0c6e8bda040e79c342d0b9867ed31 2024-11-24T08:14:13,749 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/f3c66699181f42c5b2a762cc447eab31 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/f3c66699181f42c5b2a762cc447eab31 2024-11-24T08:14:13,751 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/80345c688fb1423981f8cee307048f40 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/80345c688fb1423981f8cee307048f40 2024-11-24T08:14:13,752 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/c1785f71727c4da1a6c3a5389f70d6ad to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/c1785f71727c4da1a6c3a5389f70d6ad 2024-11-24T08:14:13,753 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/db5e5447fbea49ab92035f8927f90f5c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/db5e5447fbea49ab92035f8927f90f5c 2024-11-24T08:14:13,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/b848908a26cb4be1a307c6e10fc69260 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/b848908a26cb4be1a307c6e10fc69260 2024-11-24T08:14:13,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/807a070f6b2b47759cde355f70382322, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e75ad5ead35d47d9ab581af4726ead02, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d46077d896084e0a9097d89a85c4750b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e9f00be65a8041cb896463a331587549, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e8d0f35d441543918b4839c2d59d1747, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/ca1484b40989425a92e47fc903184027, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/7a8e3b470b5b404d8a120e97ff6cb7ed, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/1f8ffe5a520b4c7c883d967a02b6825b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/874910523ff141418535f614b5c7a654, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/c9f0380887dc414c9e22f77d0cf8d237, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5f974d2d2c994dabbd487bbdd92aad4d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/758bf47897ca463799a60de1701d5f88, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/4b93c33434874973b2f629f5da0bd678, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/3306454b3d3241b1b8e9fb55396e7a0f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/fedf639e66ff4f17a2f8d8d4a400b094, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/f44bf6b320294b66bbb4927fe131aa93, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5eb43b2e54aa44cdbd8d2da7b55c35eb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/50a8baaab9274431a5d4ad7853133181, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/638087b37d414a86a95c38a4078da98d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5f23de193a4d49038b92b2a0009f7acf, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d76f8e7055844870a3d50f317efe9306, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d756c2eacd754ded8d81089b1d549132, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d0e6b3782e8e43578dad59b1e1dbacd2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/9eab12d798b946079a758f5eb9009bc6, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/380a725ba429466880b5439b0888b74a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/864c3fb03b8b42dd9751641f35a507e4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/660f07c8bbe54545a28e892a45f4d650, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/258bceeccb8447ee91a03c2ec99e3d48, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d5c470da690e4805bcbf625cede32a7b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/9bc15faa63414091bd8f19e75baba4e3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/4b4593c1f84a48c388e14007a4c30ec3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/080e8463f5084cb0889d1a90999337e5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/b58b84988c5549bd937059b6a866ce42] to archive 2024-11-24T08:14:13,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:14:13,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/807a070f6b2b47759cde355f70382322 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/807a070f6b2b47759cde355f70382322 2024-11-24T08:14:13,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e75ad5ead35d47d9ab581af4726ead02 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e75ad5ead35d47d9ab581af4726ead02 2024-11-24T08:14:13,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d46077d896084e0a9097d89a85c4750b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d46077d896084e0a9097d89a85c4750b 2024-11-24T08:14:13,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e9f00be65a8041cb896463a331587549 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e9f00be65a8041cb896463a331587549 2024-11-24T08:14:13,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e8d0f35d441543918b4839c2d59d1747 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/e8d0f35d441543918b4839c2d59d1747 2024-11-24T08:14:13,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/ca1484b40989425a92e47fc903184027 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/ca1484b40989425a92e47fc903184027 2024-11-24T08:14:13,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/7a8e3b470b5b404d8a120e97ff6cb7ed to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/7a8e3b470b5b404d8a120e97ff6cb7ed 2024-11-24T08:14:13,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/1f8ffe5a520b4c7c883d967a02b6825b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/1f8ffe5a520b4c7c883d967a02b6825b 2024-11-24T08:14:13,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/874910523ff141418535f614b5c7a654 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/874910523ff141418535f614b5c7a654 2024-11-24T08:14:13,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/c9f0380887dc414c9e22f77d0cf8d237 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/c9f0380887dc414c9e22f77d0cf8d237 2024-11-24T08:14:13,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5f974d2d2c994dabbd487bbdd92aad4d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5f974d2d2c994dabbd487bbdd92aad4d 2024-11-24T08:14:13,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/758bf47897ca463799a60de1701d5f88 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/758bf47897ca463799a60de1701d5f88 2024-11-24T08:14:13,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/4b93c33434874973b2f629f5da0bd678 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/4b93c33434874973b2f629f5da0bd678 2024-11-24T08:14:13,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/3306454b3d3241b1b8e9fb55396e7a0f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/3306454b3d3241b1b8e9fb55396e7a0f 2024-11-24T08:14:13,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/fedf639e66ff4f17a2f8d8d4a400b094 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/fedf639e66ff4f17a2f8d8d4a400b094 2024-11-24T08:14:13,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/f44bf6b320294b66bbb4927fe131aa93 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/f44bf6b320294b66bbb4927fe131aa93 2024-11-24T08:14:13,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5eb43b2e54aa44cdbd8d2da7b55c35eb to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5eb43b2e54aa44cdbd8d2da7b55c35eb 2024-11-24T08:14:13,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/50a8baaab9274431a5d4ad7853133181 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/50a8baaab9274431a5d4ad7853133181 2024-11-24T08:14:13,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/638087b37d414a86a95c38a4078da98d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/638087b37d414a86a95c38a4078da98d 2024-11-24T08:14:13,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5f23de193a4d49038b92b2a0009f7acf to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5f23de193a4d49038b92b2a0009f7acf 2024-11-24T08:14:13,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d76f8e7055844870a3d50f317efe9306 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d76f8e7055844870a3d50f317efe9306 2024-11-24T08:14:13,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d756c2eacd754ded8d81089b1d549132 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d756c2eacd754ded8d81089b1d549132 2024-11-24T08:14:13,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d0e6b3782e8e43578dad59b1e1dbacd2 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d0e6b3782e8e43578dad59b1e1dbacd2 2024-11-24T08:14:13,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/9eab12d798b946079a758f5eb9009bc6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/9eab12d798b946079a758f5eb9009bc6 2024-11-24T08:14:13,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/380a725ba429466880b5439b0888b74a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/380a725ba429466880b5439b0888b74a 2024-11-24T08:14:13,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/864c3fb03b8b42dd9751641f35a507e4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/864c3fb03b8b42dd9751641f35a507e4 2024-11-24T08:14:13,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/660f07c8bbe54545a28e892a45f4d650 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/660f07c8bbe54545a28e892a45f4d650 2024-11-24T08:14:13,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/258bceeccb8447ee91a03c2ec99e3d48 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/258bceeccb8447ee91a03c2ec99e3d48 2024-11-24T08:14:13,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d5c470da690e4805bcbf625cede32a7b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/d5c470da690e4805bcbf625cede32a7b 2024-11-24T08:14:13,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/9bc15faa63414091bd8f19e75baba4e3 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/9bc15faa63414091bd8f19e75baba4e3 2024-11-24T08:14:13,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/4b4593c1f84a48c388e14007a4c30ec3 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/4b4593c1f84a48c388e14007a4c30ec3 2024-11-24T08:14:13,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/080e8463f5084cb0889d1a90999337e5 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/080e8463f5084cb0889d1a90999337e5 2024-11-24T08:14:13,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/b58b84988c5549bd937059b6a866ce42 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/b58b84988c5549bd937059b6a866ce42 2024-11-24T08:14:13,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/0762a25cddd54f39a2c59058e5733b04, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/120c75f2771d4716a95baa0e3d448af2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/7afe45f68a414bb5afcb2730097ea1cb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/50350840f9004f0dbec40525a5398c70, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/9734c4687f84450c8a51be381cc675ce, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/a6958a591991450c9f542afe99280680, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/1d6edc1919a041aeb50510aeef2fc32f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/5cff8a1835b747af8c9184fe2e85eda5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ef699736767d4c82b0322e968d8c199d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/f5759d0aeb7049229dfd0521be5aaf93, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/6caf7e4527a94e7ca6037ecbbde35b6a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ba753cddbc9e48aab31a8a01a7efc9af, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/cc7348022d9f4b6781ef69cce6c7ecff, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/91b7626a6e754f9e89602a2c78b73147, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/e4a22a696857466c89e3709a52a3f5b7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/160c553a9da6470991f799b4906f298b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/e01547b102ad45099a52922ecfacdc97, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/0573adb83d4540ff96ab92540999cc39, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ec8644cc7a1e435a89cdc4c75ba44a67, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/a04f64d617d242bdbcd281a4210f8f59, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/245e077c18fc49b0be085a26368ff4bb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/7b53a96d11664b2598176c7e1c16f70b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/69445ac6f9ea4e8fa8f253a19189c345, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/4200ac8cd06e400d9677fa4cac932646, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/891f7401ff3b4c0bacc4f686d99d82de, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c70d1278a0b345e09f89c0efe8f97409, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/5f7265da7ec64561a442ecdbb358e367, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/08e9460010fc46ecad115060e1abbe8c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/400b8d08b792463b86e93ec5b27e1bbc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/faf19e8eef0342439954d78eef93b4f5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/fe25fe424b984089a32ef41b08c1ea67, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/40c3abbf00d149f9b94e0d032ed9ca70, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c1057df822ab434d973bae25e6eeea7c] to archive 2024-11-24T08:14:13,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:14:13,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/0762a25cddd54f39a2c59058e5733b04 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/0762a25cddd54f39a2c59058e5733b04 2024-11-24T08:14:13,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/120c75f2771d4716a95baa0e3d448af2 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/120c75f2771d4716a95baa0e3d448af2 2024-11-24T08:14:13,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/7afe45f68a414bb5afcb2730097ea1cb to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/7afe45f68a414bb5afcb2730097ea1cb 2024-11-24T08:14:13,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/50350840f9004f0dbec40525a5398c70 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/50350840f9004f0dbec40525a5398c70 2024-11-24T08:14:13,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/9734c4687f84450c8a51be381cc675ce to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/9734c4687f84450c8a51be381cc675ce 2024-11-24T08:14:13,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/a6958a591991450c9f542afe99280680 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/a6958a591991450c9f542afe99280680 2024-11-24T08:14:13,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/1d6edc1919a041aeb50510aeef2fc32f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/1d6edc1919a041aeb50510aeef2fc32f 2024-11-24T08:14:13,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/5cff8a1835b747af8c9184fe2e85eda5 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/5cff8a1835b747af8c9184fe2e85eda5 2024-11-24T08:14:13,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ef699736767d4c82b0322e968d8c199d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ef699736767d4c82b0322e968d8c199d 2024-11-24T08:14:13,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/f5759d0aeb7049229dfd0521be5aaf93 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/f5759d0aeb7049229dfd0521be5aaf93 2024-11-24T08:14:13,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/6caf7e4527a94e7ca6037ecbbde35b6a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/6caf7e4527a94e7ca6037ecbbde35b6a 2024-11-24T08:14:13,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ba753cddbc9e48aab31a8a01a7efc9af to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ba753cddbc9e48aab31a8a01a7efc9af 2024-11-24T08:14:13,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/cc7348022d9f4b6781ef69cce6c7ecff to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/cc7348022d9f4b6781ef69cce6c7ecff 2024-11-24T08:14:13,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/91b7626a6e754f9e89602a2c78b73147 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/91b7626a6e754f9e89602a2c78b73147 2024-11-24T08:14:13,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/e4a22a696857466c89e3709a52a3f5b7 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/e4a22a696857466c89e3709a52a3f5b7 2024-11-24T08:14:13,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/160c553a9da6470991f799b4906f298b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/160c553a9da6470991f799b4906f298b 2024-11-24T08:14:13,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/e01547b102ad45099a52922ecfacdc97 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/e01547b102ad45099a52922ecfacdc97 2024-11-24T08:14:13,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/0573adb83d4540ff96ab92540999cc39 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/0573adb83d4540ff96ab92540999cc39 2024-11-24T08:14:13,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ec8644cc7a1e435a89cdc4c75ba44a67 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/ec8644cc7a1e435a89cdc4c75ba44a67 2024-11-24T08:14:13,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/a04f64d617d242bdbcd281a4210f8f59 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/a04f64d617d242bdbcd281a4210f8f59 2024-11-24T08:14:13,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/245e077c18fc49b0be085a26368ff4bb to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/245e077c18fc49b0be085a26368ff4bb 2024-11-24T08:14:13,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/7b53a96d11664b2598176c7e1c16f70b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/7b53a96d11664b2598176c7e1c16f70b 2024-11-24T08:14:13,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/69445ac6f9ea4e8fa8f253a19189c345 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/69445ac6f9ea4e8fa8f253a19189c345 2024-11-24T08:14:13,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/4200ac8cd06e400d9677fa4cac932646 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/4200ac8cd06e400d9677fa4cac932646 2024-11-24T08:14:13,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/891f7401ff3b4c0bacc4f686d99d82de to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/891f7401ff3b4c0bacc4f686d99d82de 2024-11-24T08:14:13,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c70d1278a0b345e09f89c0efe8f97409 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c70d1278a0b345e09f89c0efe8f97409 2024-11-24T08:14:13,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/5f7265da7ec64561a442ecdbb358e367 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/5f7265da7ec64561a442ecdbb358e367 2024-11-24T08:14:13,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/08e9460010fc46ecad115060e1abbe8c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/08e9460010fc46ecad115060e1abbe8c 2024-11-24T08:14:13,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/400b8d08b792463b86e93ec5b27e1bbc to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/400b8d08b792463b86e93ec5b27e1bbc 2024-11-24T08:14:13,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/faf19e8eef0342439954d78eef93b4f5 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/faf19e8eef0342439954d78eef93b4f5 2024-11-24T08:14:13,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/fe25fe424b984089a32ef41b08c1ea67 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/fe25fe424b984089a32ef41b08c1ea67 2024-11-24T08:14:13,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/40c3abbf00d149f9b94e0d032ed9ca70 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/40c3abbf00d149f9b94e0d032ed9ca70 2024-11-24T08:14:13,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c1057df822ab434d973bae25e6eeea7c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c1057df822ab434d973bae25e6eeea7c 2024-11-24T08:14:13,852 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/recovered.edits/514.seqid, newMaxSeqId=514, maxSeqId=1 2024-11-24T08:14:13,853 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. 2024-11-24T08:14:13,853 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 187d8f78b18a17144230e52cfd716831: Waiting for close lock at 1732436052453Running coprocessor pre-close hooks at 1732436052453Disabling compacts and flushes for region at 1732436052453Disabling writes for close at 1732436052453Obtaining lock to block concurrent updates at 1732436052453Preparing flush snapshotting stores in 187d8f78b18a17144230e52cfd716831 at 1732436052453Finished memstore snapshotting TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732436052454 (+1 ms)Flushing stores of TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831. at 1732436052454Flushing 187d8f78b18a17144230e52cfd716831/A: creating writer at 1732436052454Flushing 187d8f78b18a17144230e52cfd716831/A: appending metadata at 1732436052457 (+3 ms)Flushing 187d8f78b18a17144230e52cfd716831/A: closing flushed file at 1732436052458 (+1 ms)Flushing 187d8f78b18a17144230e52cfd716831/B: creating writer at 1732436052867 (+409 ms)Flushing 187d8f78b18a17144230e52cfd716831/B: appending metadata at 1732436052870 (+3 ms)Flushing 187d8f78b18a17144230e52cfd716831/B: closing flushed file at 1732436052870Flushing 187d8f78b18a17144230e52cfd716831/C: creating writer at 1732436053279 (+409 ms)Flushing 187d8f78b18a17144230e52cfd716831/C: appending metadata at 1732436053282 (+3 ms)Flushing 187d8f78b18a17144230e52cfd716831/C: closing flushed file at 1732436053282Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@463ac57c: reopening flushed file at 1732436053692 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16eddc18: reopening flushed file at 1732436053697 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ecdbb9e: reopening flushed file at 1732436053702 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 187d8f78b18a17144230e52cfd716831 in 1254ms, sequenceid=511, compaction requested=true at 1732436053707 (+5 ms)Writing region close event to WAL at 1732436053849 (+142 ms)Running coprocessor post-close hooks at 1732436053852 (+3 ms)Closed at 1732436053853 (+1 ms) 2024-11-24T08:14:13,855 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 187d8f78b18a17144230e52cfd716831 2024-11-24T08:14:13,855 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=187d8f78b18a17144230e52cfd716831, regionState=CLOSED 2024-11-24T08:14:13,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:14:13,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-24T08:14:13,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 187d8f78b18a17144230e52cfd716831, server=30c28c82771d,45087,1732436016722 in 1.5620 sec 2024-11-24T08:14:13,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-24T08:14:13,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=187d8f78b18a17144230e52cfd716831, UNASSIGN in 1.5680 sec 2024-11-24T08:14:13,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-24T08:14:13,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5750 sec 2024-11-24T08:14:13,865 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436053865"}]},"ts":"1732436053865"} 2024-11-24T08:14:13,866 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T08:14:13,866 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T08:14:13,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5890 sec 2024-11-24T08:14:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-24T08:14:14,430 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:14,433 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T08:14:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:14,438 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:14,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-24T08:14:14,440 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:14,444 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831 2024-11-24T08:14:14,447 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/recovered.edits] 2024-11-24T08:14:14,451 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/27ea5aaca22a46de93aa0e4424037612 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/27ea5aaca22a46de93aa0e4424037612 2024-11-24T08:14:14,452 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/6cbda3228dce4cd0b28243d4347f09c0 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/6cbda3228dce4cd0b28243d4347f09c0 2024-11-24T08:14:14,454 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/91912fba56964a259d29f7f979a4d57f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/A/91912fba56964a259d29f7f979a4d57f 2024-11-24T08:14:14,456 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5d64c4f708ce438bb46163cf4c313480 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/5d64c4f708ce438bb46163cf4c313480 2024-11-24T08:14:14,457 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/92da1b89c8bd45f89091cee791769bde to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/92da1b89c8bd45f89091cee791769bde 2024-11-24T08:14:14,458 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/dd2ee642bb134b95a656636f0f8f6969 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/B/dd2ee642bb134b95a656636f0f8f6969 2024-11-24T08:14:14,461 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/01abef1cdd0344939a1a4a496ffe7c0b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/01abef1cdd0344939a1a4a496ffe7c0b 2024-11-24T08:14:14,462 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/589bdd7b69d941cabb7f216f1a8b4c58 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/589bdd7b69d941cabb7f216f1a8b4c58 2024-11-24T08:14:14,463 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c3f51e1185144cf9a06ac55a5812ca1c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/C/c3f51e1185144cf9a06ac55a5812ca1c 2024-11-24T08:14:14,466 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/recovered.edits/514.seqid to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831/recovered.edits/514.seqid 2024-11-24T08:14:14,467 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/187d8f78b18a17144230e52cfd716831 2024-11-24T08:14:14,467 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T08:14:14,472 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:14,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-24T08:14:14,480 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T08:14:14,510 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T08:14:14,511 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:14,511 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T08:14:14,511 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732436054511"}]},"ts":"9223372036854775807"} 2024-11-24T08:14:14,515 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T08:14:14,515 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 187d8f78b18a17144230e52cfd716831, NAME => 'TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T08:14:14,515 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T08:14:14,516 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732436054515"}]},"ts":"9223372036854775807"} 2024-11-24T08:14:14,518 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T08:14:14,518 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:14,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 84 msec 2024-11-24T08:14:14,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-24T08:14:14,551 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T08:14:14,551 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:14,565 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/30c28c82771d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;30c28c82771d:45087-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_554043647_22 at /127.0.0.1:35778 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=445 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=274 (was 238) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7488 (was 7314) - AvailableMemoryMB LEAK? - 2024-11-24T08:14:14,576 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=274, ProcessCount=11, AvailableMemoryMB=7487 2024-11-24T08:14:14,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T08:14:14,578 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:14:14,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:14,580 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T08:14:14,580 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:14,580 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-24T08:14:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T08:14:14,581 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T08:14:14,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741945_1121 (size=1061) 2024-11-24T08:14:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T08:14:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T08:14:14,989 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6 2024-11-24T08:14:14,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741946_1122 (size=53) 2024-11-24T08:14:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T08:14:15,396 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:14:15,396 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0e4dbe9176ee2b005eb8db52027271c6, disabling compactions & flushes 2024-11-24T08:14:15,396 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:15,396 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:15,396 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. after waiting 0 ms 2024-11-24T08:14:15,396 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:15,396 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:15,396 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0e4dbe9176ee2b005eb8db52027271c6: Waiting for close lock at 1732436055396Disabling compacts and flushes for region at 1732436055396Disabling writes for close at 1732436055396Writing region close event to WAL at 1732436055396Closed at 1732436055396 2024-11-24T08:14:15,397 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T08:14:15,398 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732436055398"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732436055398"}]},"ts":"1732436055398"} 2024-11-24T08:14:15,399 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T08:14:15,400 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T08:14:15,401 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436055400"}]},"ts":"1732436055400"} 2024-11-24T08:14:15,402 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T08:14:15,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, ASSIGN}] 2024-11-24T08:14:15,403 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, ASSIGN 2024-11-24T08:14:15,404 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, ASSIGN; state=OFFLINE, location=30c28c82771d,45087,1732436016722; forceNewPlan=false, retain=false 2024-11-24T08:14:15,554 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=0e4dbe9176ee2b005eb8db52027271c6, regionState=OPENING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:14:15,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, ASSIGN because future has completed 2024-11-24T08:14:15,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:14:15,712 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:15,712 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:14:15,712 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:15,713 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:14:15,713 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:15,713 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:15,714 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:15,715 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:14:15,716 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e4dbe9176ee2b005eb8db52027271c6 columnFamilyName A 2024-11-24T08:14:15,716 DEBUG [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:15,717 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.HStore(327): Store=0e4dbe9176ee2b005eb8db52027271c6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:14:15,717 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:15,718 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:14:15,718 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e4dbe9176ee2b005eb8db52027271c6 columnFamilyName B 2024-11-24T08:14:15,718 DEBUG [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:15,718 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.HStore(327): Store=0e4dbe9176ee2b005eb8db52027271c6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:14:15,718 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:15,719 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:14:15,719 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e4dbe9176ee2b005eb8db52027271c6 columnFamilyName C 2024-11-24T08:14:15,720 DEBUG [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T08:14:15,720 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.HStore(327): Store=0e4dbe9176ee2b005eb8db52027271c6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:14:15,720 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:15,720 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:15,721 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:15,721 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:15,722 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:15,722 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:15,722 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T08:14:15,723 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:15,725 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:14:15,725 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 0e4dbe9176ee2b005eb8db52027271c6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62087013, jitterRate=-0.07483141124248505}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T08:14:15,725 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:15,726 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 0e4dbe9176ee2b005eb8db52027271c6: Running coprocessor pre-open hook at 1732436055713Writing region info on filesystem at 1732436055713Initializing all the Stores at 1732436055714 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436055714Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436055714Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436055714Cleaning up temporary data from old regions at 1732436055722 (+8 ms)Running coprocessor post-open hooks at 1732436055725 (+3 ms)Region opened successfully at 1732436055726 (+1 ms) 2024-11-24T08:14:15,727 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., pid=36, masterSystemTime=1732436055708 2024-11-24T08:14:15,728 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:15,729 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:15,729 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=0e4dbe9176ee2b005eb8db52027271c6, regionState=OPEN, openSeqNum=2, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:14:15,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:14:15,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-24T08:14:15,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 in 174 msec 2024-11-24T08:14:15,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-24T08:14:15,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, ASSIGN in 331 msec 2024-11-24T08:14:15,735 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T08:14:15,735 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436055735"}]},"ts":"1732436055735"} 2024-11-24T08:14:15,737 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T08:14:15,737 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T08:14:15,738 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1590 sec 2024-11-24T08:14:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-24T08:14:16,730 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:16,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:16,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:16,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:16,732 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388a3adb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:16,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:16,735 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:16,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a1b7e71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:16,736 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:16,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:16,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:16,740 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T08:14:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 30c28c82771d,38469,1732436015766 2024-11-24T08:14:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2b3f0ed1 2024-11-24T08:14:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T08:14:16,741 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T08:14:16,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T08:14:16,746 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:14:16,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:16,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741947_1123 (size=1097) 2024-11-24T08:14:17,167 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T08:14:17,167 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T08:14:17,171 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T08:14:17,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, REOPEN/MOVE}] 2024-11-24T08:14:17,178 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, REOPEN/MOVE 2024-11-24T08:14:17,179 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=0e4dbe9176ee2b005eb8db52027271c6, regionState=CLOSING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:14:17,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, REOPEN/MOVE because future has completed 2024-11-24T08:14:17,181 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T08:14:17,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:14:17,333 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,333 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T08:14:17,333 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 0e4dbe9176ee2b005eb8db52027271c6, disabling compactions & flushes 2024-11-24T08:14:17,333 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:17,333 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:17,333 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. after waiting 0 ms 2024-11-24T08:14:17,334 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:17,338 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T08:14:17,339 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:17,339 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 0e4dbe9176ee2b005eb8db52027271c6: Waiting for close lock at 1732436057333Running coprocessor pre-close hooks at 1732436057333Disabling compacts and flushes for region at 1732436057333Disabling writes for close at 1732436057334 (+1 ms)Writing region close event to WAL at 1732436057334Running coprocessor post-close hooks at 1732436057339 (+5 ms)Closed at 1732436057339 2024-11-24T08:14:17,339 WARN [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 0e4dbe9176ee2b005eb8db52027271c6 to self. 2024-11-24T08:14:17,341 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,342 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=0e4dbe9176ee2b005eb8db52027271c6, regionState=CLOSED 2024-11-24T08:14:17,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:14:17,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-24T08:14:17,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 in 163 msec 2024-11-24T08:14:17,346 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, REOPEN/MOVE; state=CLOSED, location=30c28c82771d,45087,1732436016722; forceNewPlan=false, retain=true 2024-11-24T08:14:17,497 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=0e4dbe9176ee2b005eb8db52027271c6, regionState=OPENING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:14:17,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, REOPEN/MOVE because future has completed 2024-11-24T08:14:17,499 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:14:17,654 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:17,654 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:14:17,654 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,654 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:14:17,654 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,655 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,659 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,659 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:14:17,664 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e4dbe9176ee2b005eb8db52027271c6 columnFamilyName A 2024-11-24T08:14:17,666 DEBUG [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:17,667 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.HStore(327): Store=0e4dbe9176ee2b005eb8db52027271c6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:14:17,667 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,668 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:14:17,668 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e4dbe9176ee2b005eb8db52027271c6 columnFamilyName B 2024-11-24T08:14:17,668 DEBUG [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:17,669 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.HStore(327): Store=0e4dbe9176ee2b005eb8db52027271c6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:14:17,669 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,669 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:14:17,670 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e4dbe9176ee2b005eb8db52027271c6 columnFamilyName C 2024-11-24T08:14:17,670 DEBUG [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:17,670 INFO [StoreOpener-0e4dbe9176ee2b005eb8db52027271c6-1 {}] regionserver.HStore(327): Store=0e4dbe9176ee2b005eb8db52027271c6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:14:17,670 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:17,670 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,671 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,672 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,673 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,673 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,673 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T08:14:17,674 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,675 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 0e4dbe9176ee2b005eb8db52027271c6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75478994, jitterRate=0.12472465634346008}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T08:14:17,675 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,677 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 0e4dbe9176ee2b005eb8db52027271c6: Running coprocessor pre-open hook at 1732436057655Writing region info on filesystem at 1732436057655Initializing all the Stores at 1732436057655Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436057655Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436057656 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436057656Cleaning up temporary data from old regions at 1732436057673 (+17 ms)Running coprocessor post-open hooks at 1732436057675 (+2 ms)Region opened successfully at 1732436057677 (+2 ms) 2024-11-24T08:14:17,678 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., pid=41, masterSystemTime=1732436057651 2024-11-24T08:14:17,680 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:17,680 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:17,680 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=0e4dbe9176ee2b005eb8db52027271c6, regionState=OPEN, openSeqNum=5, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:14:17,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:14:17,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-24T08:14:17,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 in 184 msec 2024-11-24T08:14:17,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-24T08:14:17,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, REOPEN/MOVE in 507 msec 2024-11-24T08:14:17,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-24T08:14:17,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 517 msec 2024-11-24T08:14:17,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 941 msec 2024-11-24T08:14:17,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-24T08:14:17,694 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:17,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,695 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:17,695 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:17,696 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:17,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ccb4814, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:17,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:17,698 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:17,699 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180c6c34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:17,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:17,701 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:17,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:17,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:17,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e8099a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:17,703 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62146522, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:17,704 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:17,705 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76b18d48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:17,705 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:17,706 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a0fd2ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:17,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:17,708 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67bd64ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:17,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:17,709 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:17,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d63c4d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:17,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:17,711 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64e9df0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:17,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:17,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:17,712 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:17,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de9c988, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:17,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:17,714 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:17,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:17,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:17,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b73ac8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:17,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:17,715 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:17,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4441c8b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:17,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:17,717 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51784511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:17,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:17,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:17,718 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eaadb63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,719 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:17,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:17,720 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9584eab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:17,722 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@420b069a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:17,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:17,723 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:17,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:17,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132b6c83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:17,724 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:17,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d35809d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:17,726 DEBUG [Thread-627 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:17,726 DEBUG [Thread-640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:17,726 DEBUG [Thread-631 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:17,727 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:17,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:17,727 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:17,727 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:17,728 DEBUG [Thread-623 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:17,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-24T08:14:17,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:17,729 DEBUG [Thread-634 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:17,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:17,730 DEBUG [Thread-625 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:17,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:17,730 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T08:14:17,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:17,731 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:17,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:17,731 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:17,731 DEBUG [Thread-629 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:17,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:17,731 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:17,732 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:17,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:14:17,733 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:17,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60150, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:17,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:17,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:14:17,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:17,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:14:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:14:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:17,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60178, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:14:17,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:14:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:14:17,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:14:17,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:14:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:14:17,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:17,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:17,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:17,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:17,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:17,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:17,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240b3045adaa6c4c6084951fd3d665100c_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436057743/Put/seqid=0 2024-11-24T08:14:17,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436057792, max timestamp=1732436057796Num uniques 859; 2024-11-24T08:14:17,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436057792, max timestamp=1732436057796Num uniques 859; 2024-11-24T08:14:17,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:17,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:17,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436057792, max timestamp=1732436057797Num uniques 859; 2024-11-24T08:14:17,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:17,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741948_1124 (size=17034) 2024-11-24T08:14:17,808 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:17,814 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240b3045adaa6c4c6084951fd3d665100c_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240b3045adaa6c4c6084951fd3d665100c_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:17,816 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/d981deb55fce4fc4aaf8a7eedc77f6bc, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:17,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/d981deb55fce4fc4aaf8a7eedc77f6bc is 175, key is test_row_0/A:col10/1732436057743/Put/seqid=0 2024-11-24T08:14:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T08:14:17,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741949_1125 (size=48139) 2024-11-24T08:14:17,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:17,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436117835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:17,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:17,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436117844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:17,853 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/d981deb55fce4fc4aaf8a7eedc77f6bc 2024-11-24T08:14:17,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:17,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436117845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:17,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:17,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436117848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:17,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436117849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:17,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T08:14:17,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:17,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:17,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:17,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:17,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:17,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:17,891 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/2641b472fa514874b3a1ea60f64256e5 is 50, key is test_row_0/B:col10/1732436057743/Put/seqid=0 2024-11-24T08:14:17,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741950_1126 (size=12001) 2024-11-24T08:14:17,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/2641b472fa514874b3a1ea60f64256e5 2024-11-24T08:14:17,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/e553c2f89cfb43c68f0022e6ab63e699 is 50, key is test_row_0/C:col10/1732436057743/Put/seqid=0 2024-11-24T08:14:17,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741951_1127 (size=12001) 2024-11-24T08:14:17,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/e553c2f89cfb43c68f0022e6ab63e699 2024-11-24T08:14:17,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436117950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:17,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/d981deb55fce4fc4aaf8a7eedc77f6bc as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/d981deb55fce4fc4aaf8a7eedc77f6bc 2024-11-24T08:14:17,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/d981deb55fce4fc4aaf8a7eedc77f6bc, entries=250, sequenceid=14, filesize=47.0 K 2024-11-24T08:14:17,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436117960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:17,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436117960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:17,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/2641b472fa514874b3a1ea60f64256e5 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/2641b472fa514874b3a1ea60f64256e5 2024-11-24T08:14:17,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:17,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436117961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:17,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436117960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:17,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/2641b472fa514874b3a1ea60f64256e5, entries=150, sequenceid=14, filesize=11.7 K 2024-11-24T08:14:17,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/e553c2f89cfb43c68f0022e6ab63e699 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/e553c2f89cfb43c68f0022e6ab63e699 2024-11-24T08:14:17,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/e553c2f89cfb43c68f0022e6ab63e699, entries=150, sequenceid=14, filesize=11.7 K 2024-11-24T08:14:17,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0e4dbe9176ee2b005eb8db52027271c6 in 232ms, sequenceid=14, compaction requested=false 2024-11-24T08:14:17,980 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-24T08:14:17,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:18,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-24T08:14:18,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:18,042 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-24T08:14:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:18,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T08:14:18,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249d6ffcece2e6497cb6584b5a3f65f936_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436057842/Put/seqid=0 2024-11-24T08:14:18,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741952_1128 (size=12154) 2024-11-24T08:14:18,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:18,078 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249d6ffcece2e6497cb6584b5a3f65f936_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249d6ffcece2e6497cb6584b5a3f65f936_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:18,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/3e2d6ccb9889484caca2f23da7c2e8a3, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:18,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/3e2d6ccb9889484caca2f23da7c2e8a3 is 175, key is test_row_0/A:col10/1732436057842/Put/seqid=0 2024-11-24T08:14:18,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741953_1129 (size=30955) 2024-11-24T08:14:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:18,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:18,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436118174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436118177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436118179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436118180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436118181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436118290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436118290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436118290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436118300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T08:14:18,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,501 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/3e2d6ccb9889484caca2f23da7c2e8a3 2024-11-24T08:14:18,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436118501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436118501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436118501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436118501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436118510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/de79b28572f74d7dad9fc4801b16f2fa is 50, key is test_row_0/B:col10/1732436057842/Put/seqid=0 2024-11-24T08:14:18,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741954_1130 (size=12001) 2024-11-24T08:14:18,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436118810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436118810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436118811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436118820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T08:14:18,920 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/de79b28572f74d7dad9fc4801b16f2fa 2024-11-24T08:14:18,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/46d458bdf09043cb899a172625aaa1a5 is 50, key is test_row_0/C:col10/1732436057842/Put/seqid=0 2024-11-24T08:14:18,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741955_1131 (size=12001) 2024-11-24T08:14:19,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436119024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:19,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:19,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436119321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:19,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436119321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:19,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:19,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436119321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:19,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436119340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:19,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:19,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:19,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:19,344 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/46d458bdf09043cb899a172625aaa1a5 2024-11-24T08:14:19,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/3e2d6ccb9889484caca2f23da7c2e8a3 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/3e2d6ccb9889484caca2f23da7c2e8a3 2024-11-24T08:14:19,365 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/3e2d6ccb9889484caca2f23da7c2e8a3, entries=150, sequenceid=51, filesize=30.2 K 2024-11-24T08:14:19,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/de79b28572f74d7dad9fc4801b16f2fa as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/de79b28572f74d7dad9fc4801b16f2fa 2024-11-24T08:14:19,377 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/de79b28572f74d7dad9fc4801b16f2fa, entries=150, sequenceid=51, filesize=11.7 K 2024-11-24T08:14:19,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/46d458bdf09043cb899a172625aaa1a5 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/46d458bdf09043cb899a172625aaa1a5 2024-11-24T08:14:19,416 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T08:14:19,434 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/46d458bdf09043cb899a172625aaa1a5, entries=150, sequenceid=51, filesize=11.7 K 2024-11-24T08:14:19,435 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 0e4dbe9176ee2b005eb8db52027271c6 in 1394ms, sequenceid=51, compaction requested=false 2024-11-24T08:14:19,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:19,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:19,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-24T08:14:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-24T08:14:19,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-24T08:14:19,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7050 sec 2024-11-24T08:14:19,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.7130 sec 2024-11-24T08:14:19,523 INFO [master/30c28c82771d:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-24T08:14:19,524 INFO [master/30c28c82771d:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-24T08:14:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-24T08:14:19,880 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:19,881 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:19,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-24T08:14:19,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T08:14:19,884 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:19,885 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:19,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:19,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T08:14:20,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-24T08:14:20,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:20,039 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:14:20,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:20,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:20,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:20,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:20,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:20,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:20,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:20,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:20,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bde0e807ad9b43dea519f4d28851d40a_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436058178/Put/seqid=0 2024-11-24T08:14:20,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741956_1132 (size=12154) 2024-11-24T08:14:20,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:20,123 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bde0e807ad9b43dea519f4d28851d40a_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bde0e807ad9b43dea519f4d28851d40a_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:20,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/579402c15de24308a3c990af79b72763, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:20,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/579402c15de24308a3c990af79b72763 is 175, key is test_row_0/A:col10/1732436058178/Put/seqid=0 2024-11-24T08:14:20,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741957_1133 (size=30955) 2024-11-24T08:14:20,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732436060120, max timestamp=1732436060168Num uniques 859; 2024-11-24T08:14:20,172 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732436060120, max timestamp=1732436060168Num uniques 859; 2024-11-24T08:14:20,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:14:20,173 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:14:20,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732436060120, max timestamp=1732436060168Num uniques 859; 2024-11-24T08:14:20,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:14:20,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T08:14:20,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:20,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436120263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:20,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:20,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436120330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:20,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436120330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:20,334 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in 0e4dbe9176ee2b005eb8db52027271c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:14:20,334 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 0e4dbe9176ee2b005eb8db52027271c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:14:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:20,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436120340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:20,342 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 0e4dbe9176ee2b005eb8db52027271c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:14:20,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:20,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436120350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:20,364 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 0e4dbe9176ee2b005eb8db52027271c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:14:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:20,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:20,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436120380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:20,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T08:14:20,539 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/579402c15de24308a3c990af79b72763 2024-11-24T08:14:20,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/d9564990f5954f04b6d5e27eba7afea7 is 50, key is test_row_0/B:col10/1732436058178/Put/seqid=0 2024-11-24T08:14:20,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:20,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436120590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:20,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741958_1134 (size=12001) 2024-11-24T08:14:20,603 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/d9564990f5954f04b6d5e27eba7afea7 2024-11-24T08:14:20,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/450a0cf343cf4caea24aa0cb9486f0c4 is 50, key is test_row_0/C:col10/1732436058178/Put/seqid=0 2024-11-24T08:14:20,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741959_1135 (size=12001) 2024-11-24T08:14:20,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436120910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:20,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:20,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T08:14:21,038 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/450a0cf343cf4caea24aa0cb9486f0c4 2024-11-24T08:14:21,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/579402c15de24308a3c990af79b72763 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/579402c15de24308a3c990af79b72763 2024-11-24T08:14:21,065 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/579402c15de24308a3c990af79b72763, entries=150, sequenceid=61, filesize=30.2 K 2024-11-24T08:14:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/d9564990f5954f04b6d5e27eba7afea7 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/d9564990f5954f04b6d5e27eba7afea7 2024-11-24T08:14:21,071 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/d9564990f5954f04b6d5e27eba7afea7, entries=150, sequenceid=61, filesize=11.7 K 2024-11-24T08:14:21,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/450a0cf343cf4caea24aa0cb9486f0c4 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/450a0cf343cf4caea24aa0cb9486f0c4 2024-11-24T08:14:21,078 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/450a0cf343cf4caea24aa0cb9486f0c4, entries=150, sequenceid=61, filesize=11.7 K 2024-11-24T08:14:21,079 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 0e4dbe9176ee2b005eb8db52027271c6 in 1040ms, sequenceid=61, compaction requested=true 2024-11-24T08:14:21,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:21,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:21,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-24T08:14:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-24T08:14:21,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-24T08:14:21,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1960 sec 2024-11-24T08:14:21,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.2030 sec 2024-11-24T08:14:21,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-24T08:14:21,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:21,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:21,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:21,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:21,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:21,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:21,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e4f6d283f9f74b6586c24a09a29efa7a_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436060240/Put/seqid=0 2024-11-24T08:14:21,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436121482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:21,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:21,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:21,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:21,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741960_1136 (size=14594) 2024-11-24T08:14:21,504 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:21,512 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e4f6d283f9f74b6586c24a09a29efa7a_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e4f6d283f9f74b6586c24a09a29efa7a_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:21,513 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/67ddba911bf947e2896026f575a23fa0, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:21,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/67ddba911bf947e2896026f575a23fa0 is 175, key is test_row_0/A:col10/1732436060240/Put/seqid=0 2024-11-24T08:14:21,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741961_1137 (size=39549) 2024-11-24T08:14:21,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436121600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:21,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:21,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436121820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:21,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:21,929 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/67ddba911bf947e2896026f575a23fa0 2024-11-24T08:14:21,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/ebff6f54ff6f4d37be2b39284a4d7cdd is 50, key is test_row_0/B:col10/1732436060240/Put/seqid=0 2024-11-24T08:14:21,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741962_1138 (size=12001) 2024-11-24T08:14:21,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/ebff6f54ff6f4d37be2b39284a4d7cdd 2024-11-24T08:14:21,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/e2be7efe32174d0b9c54942ed87c180d is 50, key is test_row_0/C:col10/1732436060240/Put/seqid=0 2024-11-24T08:14:21,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741963_1139 (size=12001) 2024-11-24T08:14:21,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/e2be7efe32174d0b9c54942ed87c180d 2024-11-24T08:14:21,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/67ddba911bf947e2896026f575a23fa0 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/67ddba911bf947e2896026f575a23fa0 2024-11-24T08:14:21,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/67ddba911bf947e2896026f575a23fa0, entries=200, sequenceid=98, filesize=38.6 K 2024-11-24T08:14:21,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/ebff6f54ff6f4d37be2b39284a4d7cdd as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ebff6f54ff6f4d37be2b39284a4d7cdd 2024-11-24T08:14:22,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ebff6f54ff6f4d37be2b39284a4d7cdd, entries=150, sequenceid=98, filesize=11.7 K 2024-11-24T08:14:22,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/e2be7efe32174d0b9c54942ed87c180d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/e2be7efe32174d0b9c54942ed87c180d 2024-11-24T08:14:22,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/e2be7efe32174d0b9c54942ed87c180d, entries=150, sequenceid=98, filesize=11.7 K 2024-11-24T08:14:22,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 0e4dbe9176ee2b005eb8db52027271c6 in 580ms, sequenceid=98, compaction requested=true 2024-11-24T08:14:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:22,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:22,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:22,014 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:14:22,014 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:14:22,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:22,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:22,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:22,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:22,016 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:14:22,016 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/A is initiating minor compaction (all files) 2024-11-24T08:14:22,016 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/A in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,016 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/d981deb55fce4fc4aaf8a7eedc77f6bc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/3e2d6ccb9889484caca2f23da7c2e8a3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/579402c15de24308a3c990af79b72763, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/67ddba911bf947e2896026f575a23fa0] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=146.1 K 2024-11-24T08:14:22,016 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,016 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/d981deb55fce4fc4aaf8a7eedc77f6bc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/3e2d6ccb9889484caca2f23da7c2e8a3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/579402c15de24308a3c990af79b72763, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/67ddba911bf947e2896026f575a23fa0] 2024-11-24T08:14:22,018 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:14:22,018 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/B is initiating minor compaction (all files) 2024-11-24T08:14:22,018 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/B in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,018 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/2641b472fa514874b3a1ea60f64256e5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/de79b28572f74d7dad9fc4801b16f2fa, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/d9564990f5954f04b6d5e27eba7afea7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ebff6f54ff6f4d37be2b39284a4d7cdd] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=46.9 K 2024-11-24T08:14:22,019 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d981deb55fce4fc4aaf8a7eedc77f6bc, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732436057739 2024-11-24T08:14:22,019 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2641b472fa514874b3a1ea60f64256e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732436057743 2024-11-24T08:14:22,019 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e2d6ccb9889484caca2f23da7c2e8a3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732436057839 2024-11-24T08:14:22,020 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting de79b28572f74d7dad9fc4801b16f2fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732436057839 2024-11-24T08:14:22,021 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 579402c15de24308a3c990af79b72763, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732436058171 2024-11-24T08:14:22,021 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting d9564990f5954f04b6d5e27eba7afea7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732436058171 2024-11-24T08:14:22,021 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67ddba911bf947e2896026f575a23fa0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732436060219 2024-11-24T08:14:22,022 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting ebff6f54ff6f4d37be2b39284a4d7cdd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732436060240 2024-11-24T08:14:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-24T08:14:22,031 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:22,032 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-24T08:14:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T08:14:22,040 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:22,042 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:22,042 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:22,052 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:22,058 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#B#compaction#121 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:22,058 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/7af74a6ff6814ba2a3b12be97bf22c5d is 50, key is test_row_0/B:col10/1732436060240/Put/seqid=0 2024-11-24T08:14:22,072 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124bc1c5747a5554933ae7d1083a17f7581_0e4dbe9176ee2b005eb8db52027271c6 store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:22,077 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124bc1c5747a5554933ae7d1083a17f7581_0e4dbe9176ee2b005eb8db52027271c6, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:22,077 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bc1c5747a5554933ae7d1083a17f7581_0e4dbe9176ee2b005eb8db52027271c6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:22,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741964_1140 (size=12139) 2024-11-24T08:14:22,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:14:22,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:22,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:22,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:22,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:22,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:22,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T08:14:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:22,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741965_1141 (size=4469) 2024-11-24T08:14:22,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246c1b25383b5e44c49a14efa895296768_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436062145/Put/seqid=0 2024-11-24T08:14:22,165 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#A#compaction#120 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:22,165 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/75b3224c141440ddaf4c52216c70bc89 is 175, key is test_row_0/A:col10/1732436060240/Put/seqid=0 2024-11-24T08:14:22,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T08:14:22,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:22,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741966_1142 (size=14594) 2024-11-24T08:14:22,217 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:22,231 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246c1b25383b5e44c49a14efa895296768_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246c1b25383b5e44c49a14efa895296768_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:22,232 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/1d0507764916430e9ceabf78c4a32270, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:22,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/1d0507764916430e9ceabf78c4a32270 is 175, key is test_row_0/A:col10/1732436062145/Put/seqid=0 2024-11-24T08:14:22,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741967_1143 (size=31093) 2024-11-24T08:14:22,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741968_1144 (size=39549) 2024-11-24T08:14:22,272 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/1d0507764916430e9ceabf78c4a32270 2024-11-24T08:14:22,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732436062209, max timestamp=1732436062291Num uniques 859; 2024-11-24T08:14:22,295 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:14:22,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436062267, max timestamp=1732436062291Num uniques 859; 2024-11-24T08:14:22,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:22,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436062267, max timestamp=1732436062291Num uniques 859; 2024-11-24T08:14:22,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:22,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/7fef601ef21242efb80d14dc27fabc60 is 50, key is test_row_0/B:col10/1732436062145/Put/seqid=0 2024-11-24T08:14:22,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741969_1145 (size=12001) 2024-11-24T08:14:22,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T08:14:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:22,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/7fef601ef21242efb80d14dc27fabc60 2024-11-24T08:14:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T08:14:22,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/a3726ce5ce744a8d93012887ebbc334a is 50, key is test_row_0/C:col10/1732436062145/Put/seqid=0 2024-11-24T08:14:22,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741970_1146 (size=12001) 2024-11-24T08:14:22,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436122386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436122388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436122394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,403 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436122394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436122395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T08:14:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436122510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436122511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436122511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436122511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436122511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,526 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/7af74a6ff6814ba2a3b12be97bf22c5d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7af74a6ff6814ba2a3b12be97bf22c5d 2024-11-24T08:14:22,557 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/B of 0e4dbe9176ee2b005eb8db52027271c6 into 7af74a6ff6814ba2a3b12be97bf22c5d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:22,557 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:22,557 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/B, priority=12, startTime=1732436062014; duration=0sec 2024-11-24T08:14:22,557 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:22,557 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:B 2024-11-24T08:14:22,558 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:14:22,560 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:14:22,560 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/C is initiating minor compaction (all files) 2024-11-24T08:14:22,561 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/C in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,561 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/e553c2f89cfb43c68f0022e6ab63e699, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/46d458bdf09043cb899a172625aaa1a5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/450a0cf343cf4caea24aa0cb9486f0c4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/e2be7efe32174d0b9c54942ed87c180d] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=46.9 K 2024-11-24T08:14:22,561 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting e553c2f89cfb43c68f0022e6ab63e699, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732436057743 2024-11-24T08:14:22,562 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 46d458bdf09043cb899a172625aaa1a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732436057839 2024-11-24T08:14:22,562 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 450a0cf343cf4caea24aa0cb9486f0c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732436058171 2024-11-24T08:14:22,563 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting e2be7efe32174d0b9c54942ed87c180d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732436060240 2024-11-24T08:14:22,587 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:22,588 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/c5b49f12942842328361301c6c070100 is 50, key is test_row_0/C:col10/1732436060240/Put/seqid=0 2024-11-24T08:14:22,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741971_1147 (size=12139) 2024-11-24T08:14:22,638 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/c5b49f12942842328361301c6c070100 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/c5b49f12942842328361301c6c070100 2024-11-24T08:14:22,649 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/75b3224c141440ddaf4c52216c70bc89 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/75b3224c141440ddaf4c52216c70bc89 2024-11-24T08:14:22,656 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/A of 0e4dbe9176ee2b005eb8db52027271c6 into 75b3224c141440ddaf4c52216c70bc89(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:22,656 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:22,656 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/A, priority=12, startTime=1732436062014; duration=0sec 2024-11-24T08:14:22,657 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:22,657 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:A 2024-11-24T08:14:22,664 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/C of 0e4dbe9176ee2b005eb8db52027271c6 into c5b49f12942842328361301c6c070100(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:22,664 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:22,664 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/C, priority=12, startTime=1732436062014; duration=0sec 2024-11-24T08:14:22,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T08:14:22,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,665 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:22,665 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:C 2024-11-24T08:14:22,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:22,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T08:14:22,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436122720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436122720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436122720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436122720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436122730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:22,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:22,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/a3726ce5ce744a8d93012887ebbc334a 2024-11-24T08:14:22,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/1d0507764916430e9ceabf78c4a32270 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/1d0507764916430e9ceabf78c4a32270 2024-11-24T08:14:22,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/1d0507764916430e9ceabf78c4a32270, entries=200, sequenceid=108, filesize=38.6 K 2024-11-24T08:14:22,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/7fef601ef21242efb80d14dc27fabc60 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7fef601ef21242efb80d14dc27fabc60 2024-11-24T08:14:22,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7fef601ef21242efb80d14dc27fabc60, entries=150, sequenceid=108, filesize=11.7 K 2024-11-24T08:14:22,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T08:14:22,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:22,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/a3726ce5ce744a8d93012887ebbc334a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/a3726ce5ce744a8d93012887ebbc334a 2024-11-24T08:14:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:22,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/a3726ce5ce744a8d93012887ebbc334a, entries=150, sequenceid=108, filesize=11.7 K 2024-11-24T08:14:22,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 0e4dbe9176ee2b005eb8db52027271c6 in 685ms, sequenceid=108, compaction requested=false 2024-11-24T08:14:22,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:22,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-24T08:14:22,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:22,978 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=136.42 KB heapSize=390.06 KB 2024-11-24T08:14:22,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:22,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:22,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:22,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:22,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:22,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:23,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244bd51b6cccfd4cbfbc43f2f0d0115add_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436062300/Put/seqid=0 2024-11-24T08:14:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:23,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:23,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741972_1148 (size=12304) 2024-11-24T08:14:23,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:23,056 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244bd51b6cccfd4cbfbc43f2f0d0115add_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244bd51b6cccfd4cbfbc43f2f0d0115add_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:23,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/d80c197c79254489a6b9c080bacfcb0f, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:23,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/d80c197c79254489a6b9c080bacfcb0f is 175, key is test_row_0/A:col10/1732436062300/Put/seqid=0 2024-11-24T08:14:23,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436123050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436123052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436123057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436123050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436123064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741973_1149 (size=31105) 2024-11-24T08:14:23,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T08:14:23,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436123181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436123181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436123184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436123184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436123184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436123390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436123390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,394 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436123390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436123400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436123400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,490 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/d80c197c79254489a6b9c080bacfcb0f 2024-11-24T08:14:23,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/25433e6fe0ff4757bb550bd603ac9157 is 50, key is test_row_0/B:col10/1732436062300/Put/seqid=0 2024-11-24T08:14:23,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741974_1150 (size=12151) 2024-11-24T08:14:23,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436123701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436123701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,707 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436123701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436123710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436123710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:23,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:23,984 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/25433e6fe0ff4757bb550bd603ac9157 2024-11-24T08:14:23,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/9e2498122e42475bad29dd47bda094b1 is 50, key is test_row_0/C:col10/1732436062300/Put/seqid=0 2024-11-24T08:14:24,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741975_1151 (size=12151) 2024-11-24T08:14:24,000 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/9e2498122e42475bad29dd47bda094b1 2024-11-24T08:14:24,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/d80c197c79254489a6b9c080bacfcb0f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/d80c197c79254489a6b9c080bacfcb0f 2024-11-24T08:14:24,022 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/d80c197c79254489a6b9c080bacfcb0f, entries=150, sequenceid=148, filesize=30.4 K 2024-11-24T08:14:24,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/25433e6fe0ff4757bb550bd603ac9157 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/25433e6fe0ff4757bb550bd603ac9157 2024-11-24T08:14:24,040 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/25433e6fe0ff4757bb550bd603ac9157, entries=150, sequenceid=148, filesize=11.9 K 2024-11-24T08:14:24,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/9e2498122e42475bad29dd47bda094b1 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/9e2498122e42475bad29dd47bda094b1 2024-11-24T08:14:24,050 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/9e2498122e42475bad29dd47bda094b1, entries=150, sequenceid=148, filesize=11.9 K 2024-11-24T08:14:24,051 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~390.02 KB/399376, currentSize=46.96 KB/48090 for 0e4dbe9176ee2b005eb8db52027271c6 in 1073ms, sequenceid=148, compaction requested=true 2024-11-24T08:14:24,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:24,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:24,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-24T08:14:24,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-24T08:14:24,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-24T08:14:24,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0110 sec 2024-11-24T08:14:24,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.0230 sec 2024-11-24T08:14:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-24T08:14:24,190 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:24,190 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:24,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-24T08:14:24,194 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T08:14:24,195 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:24,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:24,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T08:14:24,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:24,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:24,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:24,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:24,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:24,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:24,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249507726cf65c418d956c0c5918c07ce2_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436064224/Put/seqid=0 2024-11-24T08:14:24,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741976_1152 (size=17284) 2024-11-24T08:14:24,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436124250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436124251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,260 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436124256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436124257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436124258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T08:14:24,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T08:14:24,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:24,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:24,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:24,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:24,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:24,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436124370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436124370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436124370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436124371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436124371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T08:14:24,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:24,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:24,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T08:14:24,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436124580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436124580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436124580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436124581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436124581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,649 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:24,655 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249507726cf65c418d956c0c5918c07ce2_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249507726cf65c418d956c0c5918c07ce2_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:24,656 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/46c9b65f426847ffb18dd8903bba474f, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:24,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/46c9b65f426847ffb18dd8903bba474f is 175, key is test_row_0/A:col10/1732436064224/Put/seqid=0 2024-11-24T08:14:24,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T08:14:24,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:24,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:24,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:24,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:24,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:24,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741977_1153 (size=48389) 2024-11-24T08:14:24,666 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/46c9b65f426847ffb18dd8903bba474f 2024-11-24T08:14:24,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/7f21d489f42a4e178825cf3916f5beec is 50, key is test_row_0/B:col10/1732436064224/Put/seqid=0 2024-11-24T08:14:24,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741978_1154 (size=12151) 2024-11-24T08:14:24,682 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/7f21d489f42a4e178825cf3916f5beec 2024-11-24T08:14:24,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/3f5bc05d64f047c08911ac0dce4e79fe is 50, key is test_row_0/C:col10/1732436064224/Put/seqid=0 2024-11-24T08:14:24,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741979_1155 (size=12151) 2024-11-24T08:14:24,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/3f5bc05d64f047c08911ac0dce4e79fe 2024-11-24T08:14:24,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/46c9b65f426847ffb18dd8903bba474f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/46c9b65f426847ffb18dd8903bba474f 2024-11-24T08:14:24,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/46c9b65f426847ffb18dd8903bba474f, entries=250, sequenceid=161, filesize=47.3 K 2024-11-24T08:14:24,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/7f21d489f42a4e178825cf3916f5beec as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7f21d489f42a4e178825cf3916f5beec 2024-11-24T08:14:24,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7f21d489f42a4e178825cf3916f5beec, entries=150, sequenceid=161, filesize=11.9 K 2024-11-24T08:14:24,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/3f5bc05d64f047c08911ac0dce4e79fe as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/3f5bc05d64f047c08911ac0dce4e79fe 2024-11-24T08:14:24,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/3f5bc05d64f047c08911ac0dce4e79fe, entries=150, sequenceid=161, filesize=11.9 K 2024-11-24T08:14:24,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 0e4dbe9176ee2b005eb8db52027271c6 in 525ms, sequenceid=161, compaction requested=true 2024-11-24T08:14:24,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:24,750 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:14:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T08:14:24,750 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:14:24,752 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150136 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:14:24,752 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/A is initiating minor compaction (all files) 2024-11-24T08:14:24,752 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/A in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:24,752 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/75b3224c141440ddaf4c52216c70bc89, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/1d0507764916430e9ceabf78c4a32270, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/d80c197c79254489a6b9c080bacfcb0f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/46c9b65f426847ffb18dd8903bba474f] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=146.6 K 2024-11-24T08:14:24,752 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:24,752 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:14:24,752 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/75b3224c141440ddaf4c52216c70bc89, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/1d0507764916430e9ceabf78c4a32270, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/d80c197c79254489a6b9c080bacfcb0f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/46c9b65f426847ffb18dd8903bba474f] 2024-11-24T08:14:24,752 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/B is initiating minor compaction (all files) 2024-11-24T08:14:24,752 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/B in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:24,753 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7af74a6ff6814ba2a3b12be97bf22c5d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7fef601ef21242efb80d14dc27fabc60, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/25433e6fe0ff4757bb550bd603ac9157, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7f21d489f42a4e178825cf3916f5beec] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=47.3 K 2024-11-24T08:14:24,753 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75b3224c141440ddaf4c52216c70bc89, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732436060240 2024-11-24T08:14:24,757 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7af74a6ff6814ba2a3b12be97bf22c5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732436060240 2024-11-24T08:14:24,757 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d0507764916430e9ceabf78c4a32270, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732436061469 2024-11-24T08:14:24,758 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fef601ef21242efb80d14dc27fabc60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732436061469 2024-11-24T08:14:24,758 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d80c197c79254489a6b9c080bacfcb0f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732436062300 2024-11-24T08:14:24,760 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46c9b65f426847ffb18dd8903bba474f, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732436063049 2024-11-24T08:14:24,760 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 25433e6fe0ff4757bb550bd603ac9157, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732436062300 2024-11-24T08:14:24,761 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f21d489f42a4e178825cf3916f5beec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732436063049 2024-11-24T08:14:24,777 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:24,779 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:24,779 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/95b63fdb3d60485aa85ec33b95011c14 is 50, key is test_row_0/B:col10/1732436064224/Put/seqid=0 2024-11-24T08:14:24,789 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124b08e3270ae8e49ed87be1831c56b4aa0_0e4dbe9176ee2b005eb8db52027271c6 store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:24,792 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124b08e3270ae8e49ed87be1831c56b4aa0_0e4dbe9176ee2b005eb8db52027271c6, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:24,793 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b08e3270ae8e49ed87be1831c56b4aa0_0e4dbe9176ee2b005eb8db52027271c6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:24,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741980_1156 (size=12425) 2024-11-24T08:14:24,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741981_1157 (size=4469) 2024-11-24T08:14:24,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-24T08:14:24,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:24,814 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T08:14:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:24,816 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#A#compaction#132 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:24,816 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/ccf4eca1de4f4bcca553adc24e78378d is 175, key is test_row_0/A:col10/1732436064224/Put/seqid=0 2024-11-24T08:14:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T08:14:24,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741982_1158 (size=31379) 2024-11-24T08:14:24,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112477e932616e524e23b6e2fe208a0248bd_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436064256/Put/seqid=0 2024-11-24T08:14:24,841 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/ccf4eca1de4f4bcca553adc24e78378d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/ccf4eca1de4f4bcca553adc24e78378d 2024-11-24T08:14:24,851 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/A of 0e4dbe9176ee2b005eb8db52027271c6 into ccf4eca1de4f4bcca553adc24e78378d(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:24,851 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:24,851 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/A, priority=12, startTime=1732436064750; duration=0sec 2024-11-24T08:14:24,851 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:24,852 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:A 2024-11-24T08:14:24,852 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:14:24,853 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:14:24,853 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/C is initiating minor compaction (all files) 2024-11-24T08:14:24,853 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/C in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:24,853 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/c5b49f12942842328361301c6c070100, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/a3726ce5ce744a8d93012887ebbc334a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/9e2498122e42475bad29dd47bda094b1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/3f5bc05d64f047c08911ac0dce4e79fe] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=47.3 K 2024-11-24T08:14:24,854 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5b49f12942842328361301c6c070100, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1732436060240 2024-11-24T08:14:24,855 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3726ce5ce744a8d93012887ebbc334a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732436061469 2024-11-24T08:14:24,855 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e2498122e42475bad29dd47bda094b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1732436062300 2024-11-24T08:14:24,855 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f5bc05d64f047c08911ac0dce4e79fe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732436063049 2024-11-24T08:14:24,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741983_1159 (size=12304) 2024-11-24T08:14:24,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:24,870 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112477e932616e524e23b6e2fe208a0248bd_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112477e932616e524e23b6e2fe208a0248bd_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:24,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/2f17da791b4e4eb696ec177eeeff338b, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/2f17da791b4e4eb696ec177eeeff338b is 175, key is test_row_0/A:col10/1732436064256/Put/seqid=0 2024-11-24T08:14:24,882 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#C#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:24,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741984_1160 (size=31105) 2024-11-24T08:14:24,883 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/23b3fb706a18422b8508836f4d3e4c96 is 50, key is test_row_0/C:col10/1732436064224/Put/seqid=0 2024-11-24T08:14:24,884 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=181, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/2f17da791b4e4eb696ec177eeeff338b 2024-11-24T08:14:24,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:24,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:24,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741985_1161 (size=12425) 2024-11-24T08:14:24,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/7d9c7a47019945c0b813fd602de6d1e4 is 50, key is test_row_0/B:col10/1732436064256/Put/seqid=0 2024-11-24T08:14:24,910 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/23b3fb706a18422b8508836f4d3e4c96 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/23b3fb706a18422b8508836f4d3e4c96 2024-11-24T08:14:24,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741986_1162 (size=12151) 2024-11-24T08:14:24,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436124902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,912 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436124905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,918 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/C of 0e4dbe9176ee2b005eb8db52027271c6 into 23b3fb706a18422b8508836f4d3e4c96(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:24,918 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:24,918 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/C, priority=12, startTime=1732436064750; duration=0sec 2024-11-24T08:14:24,918 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:24,918 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:C 2024-11-24T08:14:24,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436124907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:24,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436124911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436124913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:24,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436125020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436125020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436125029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436125029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436125030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,212 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/95b63fdb3d60485aa85ec33b95011c14 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/95b63fdb3d60485aa85ec33b95011c14 2024-11-24T08:14:25,225 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/B of 0e4dbe9176ee2b005eb8db52027271c6 into 95b63fdb3d60485aa85ec33b95011c14(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:25,225 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:25,225 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/B, priority=12, startTime=1732436064750; duration=0sec 2024-11-24T08:14:25,226 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:25,226 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:B 2024-11-24T08:14:25,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436125230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436125231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436125240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436125240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436125240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,313 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/7d9c7a47019945c0b813fd602de6d1e4 2024-11-24T08:14:25,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/5276fe22353547c59e9e2a4ded431846 is 50, key is test_row_0/C:col10/1732436064256/Put/seqid=0 2024-11-24T08:14:25,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T08:14:25,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741987_1163 (size=12151) 2024-11-24T08:14:25,355 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/5276fe22353547c59e9e2a4ded431846 2024-11-24T08:14:25,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/2f17da791b4e4eb696ec177eeeff338b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/2f17da791b4e4eb696ec177eeeff338b 2024-11-24T08:14:25,369 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/2f17da791b4e4eb696ec177eeeff338b, entries=150, sequenceid=181, filesize=30.4 K 2024-11-24T08:14:25,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/7d9c7a47019945c0b813fd602de6d1e4 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7d9c7a47019945c0b813fd602de6d1e4 2024-11-24T08:14:25,377 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7d9c7a47019945c0b813fd602de6d1e4, entries=150, sequenceid=181, filesize=11.9 K 2024-11-24T08:14:25,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/5276fe22353547c59e9e2a4ded431846 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/5276fe22353547c59e9e2a4ded431846 2024-11-24T08:14:25,397 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/5276fe22353547c59e9e2a4ded431846, entries=150, sequenceid=181, filesize=11.9 K 2024-11-24T08:14:25,398 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 0e4dbe9176ee2b005eb8db52027271c6 in 584ms, sequenceid=181, compaction requested=false 2024-11-24T08:14:25,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:25,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:25,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-24T08:14:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-24T08:14:25,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-24T08:14:25,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2050 sec 2024-11-24T08:14:25,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.2160 sec 2024-11-24T08:14:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:25,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T08:14:25,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:25,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:25,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:25,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:25,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:25,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:25,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e203ac9536494ccc80f33630ad924c6f_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436064905/Put/seqid=0 2024-11-24T08:14:25,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741988_1164 (size=12304) 2024-11-24T08:14:25,585 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:25,591 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e203ac9536494ccc80f33630ad924c6f_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e203ac9536494ccc80f33630ad924c6f_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:25,592 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/15a79822e4df4adf8352c38853f567f0, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:25,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/15a79822e4df4adf8352c38853f567f0 is 175, key is test_row_0/A:col10/1732436064905/Put/seqid=0 2024-11-24T08:14:25,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741989_1165 (size=31105) 2024-11-24T08:14:25,614 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/15a79822e4df4adf8352c38853f567f0 2024-11-24T08:14:25,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436125623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436125624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/899268ab3ab04235b49d1f40333001af is 50, key is test_row_0/B:col10/1732436064905/Put/seqid=0 2024-11-24T08:14:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436125630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436125633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436125639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741990_1166 (size=12151) 2024-11-24T08:14:25,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436125740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436125751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436125751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436125751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436125751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436125950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436125960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436125970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436125970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:25,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436125970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:25,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:25,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/899268ab3ab04235b49d1f40333001af 2024-11-24T08:14:26,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/8e15fb1f717748c4b1fbc643e6d8eaac is 50, key is test_row_0/C:col10/1732436064905/Put/seqid=0 2024-11-24T08:14:26,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741991_1167 (size=12151) 2024-11-24T08:14:26,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/8e15fb1f717748c4b1fbc643e6d8eaac 2024-11-24T08:14:26,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/15a79822e4df4adf8352c38853f567f0 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/15a79822e4df4adf8352c38853f567f0 2024-11-24T08:14:26,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/15a79822e4df4adf8352c38853f567f0, entries=150, sequenceid=197, filesize=30.4 K 2024-11-24T08:14:26,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/899268ab3ab04235b49d1f40333001af as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/899268ab3ab04235b49d1f40333001af 2024-11-24T08:14:26,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/899268ab3ab04235b49d1f40333001af, entries=150, sequenceid=197, filesize=11.9 K 2024-11-24T08:14:26,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/8e15fb1f717748c4b1fbc643e6d8eaac as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/8e15fb1f717748c4b1fbc643e6d8eaac 2024-11-24T08:14:26,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/8e15fb1f717748c4b1fbc643e6d8eaac, entries=150, sequenceid=197, filesize=11.9 K 2024-11-24T08:14:26,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 0e4dbe9176ee2b005eb8db52027271c6 in 644ms, sequenceid=197, compaction requested=true 2024-11-24T08:14:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:26,187 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:26,187 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:26,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:26,189 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:26,189 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/A is initiating minor compaction (all files) 2024-11-24T08:14:26,189 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/A in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:26,189 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/ccf4eca1de4f4bcca553adc24e78378d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/2f17da791b4e4eb696ec177eeeff338b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/15a79822e4df4adf8352c38853f567f0] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=91.4 K 2024-11-24T08:14:26,189 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:26,189 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/ccf4eca1de4f4bcca553adc24e78378d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/2f17da791b4e4eb696ec177eeeff338b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/15a79822e4df4adf8352c38853f567f0] 2024-11-24T08:14:26,189 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:26,190 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/B is initiating minor compaction (all files) 2024-11-24T08:14:26,190 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/B in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:26,190 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/95b63fdb3d60485aa85ec33b95011c14, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7d9c7a47019945c0b813fd602de6d1e4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/899268ab3ab04235b49d1f40333001af] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=35.9 K 2024-11-24T08:14:26,190 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccf4eca1de4f4bcca553adc24e78378d, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732436063049 2024-11-24T08:14:26,191 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 95b63fdb3d60485aa85ec33b95011c14, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732436063049 2024-11-24T08:14:26,191 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f17da791b4e4eb696ec177eeeff338b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732436064249 2024-11-24T08:14:26,192 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15a79822e4df4adf8352c38853f567f0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732436064905 2024-11-24T08:14:26,192 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d9c7a47019945c0b813fd602de6d1e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732436064249 2024-11-24T08:14:26,193 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 899268ab3ab04235b49d1f40333001af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732436064905 2024-11-24T08:14:26,214 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:26,216 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#B#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:26,217 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/adc3e825e3034cda81ac6b260c6b68a1 is 50, key is test_row_0/B:col10/1732436064905/Put/seqid=0 2024-11-24T08:14:26,218 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124fffbee433ba141a88d2faa5ee67aa0b5_0e4dbe9176ee2b005eb8db52027271c6 store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:26,224 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124fffbee433ba141a88d2faa5ee67aa0b5_0e4dbe9176ee2b005eb8db52027271c6, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:26,225 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124fffbee433ba141a88d2faa5ee67aa0b5_0e4dbe9176ee2b005eb8db52027271c6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:26,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741992_1168 (size=4469) 2024-11-24T08:14:26,270 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#A#compaction#142 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:26,271 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/35ea893af27e4b65933065beb710179f is 175, key is test_row_0/A:col10/1732436064905/Put/seqid=0 2024-11-24T08:14:26,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741993_1169 (size=12527) 2024-11-24T08:14:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:26,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T08:14:26,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:26,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:26,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:26,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:26,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:26,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:26,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436126285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436126286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436126286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436126287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436126295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741994_1170 (size=31481) 2024-11-24T08:14:26,315 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/35ea893af27e4b65933065beb710179f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/35ea893af27e4b65933065beb710179f 2024-11-24T08:14:26,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e05ee036e9754765a2f04954449fe7de_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436066275/Put/seqid=0 2024-11-24T08:14:26,333 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/A of 0e4dbe9176ee2b005eb8db52027271c6 into 35ea893af27e4b65933065beb710179f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:26,333 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:26,333 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/A, priority=13, startTime=1732436066187; duration=0sec 2024-11-24T08:14:26,333 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:26,333 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:A 2024-11-24T08:14:26,333 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:26,334 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:26,334 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/C is initiating minor compaction (all files) 2024-11-24T08:14:26,334 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/C in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:26,335 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/23b3fb706a18422b8508836f4d3e4c96, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/5276fe22353547c59e9e2a4ded431846, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/8e15fb1f717748c4b1fbc643e6d8eaac] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=35.9 K 2024-11-24T08:14:26,335 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23b3fb706a18422b8508836f4d3e4c96, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732436063049 2024-11-24T08:14:26,335 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5276fe22353547c59e9e2a4ded431846, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1732436064249 2024-11-24T08:14:26,336 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e15fb1f717748c4b1fbc643e6d8eaac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732436064905 2024-11-24T08:14:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-24T08:14:26,352 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:26,353 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-24T08:14:26,357 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:26,358 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:26,358 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T08:14:26,363 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#C#compaction#144 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:26,364 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/51ca97b53d754dc9bcb928f83a4e84de is 50, key is test_row_0/C:col10/1732436064905/Put/seqid=0 2024-11-24T08:14:26,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741995_1171 (size=17284) 2024-11-24T08:14:26,379 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:26,386 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e05ee036e9754765a2f04954449fe7de_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e05ee036e9754765a2f04954449fe7de_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:26,387 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/11b086ce04324f58b08b7421c5bed46f, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:26,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/11b086ce04324f58b08b7421c5bed46f is 175, key is test_row_0/A:col10/1732436066275/Put/seqid=0 2024-11-24T08:14:26,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436126400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436126409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436126410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436126410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436126410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741996_1172 (size=12527) 2024-11-24T08:14:26,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741997_1173 (size=48389) 2024-11-24T08:14:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T08:14:26,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T08:14:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:26,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:26,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:26,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436126610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436126630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436126630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436126632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,642 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436126640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T08:14:26,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:26,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:26,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:26,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:26,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T08:14:26,692 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/adc3e825e3034cda81ac6b260c6b68a1 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/adc3e825e3034cda81ac6b260c6b68a1 2024-11-24T08:14:26,702 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/B of 0e4dbe9176ee2b005eb8db52027271c6 into adc3e825e3034cda81ac6b260c6b68a1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:26,702 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:26,702 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/B, priority=13, startTime=1732436066187; duration=0sec 2024-11-24T08:14:26,702 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:26,702 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:B 2024-11-24T08:14:26,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T08:14:26,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:26,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:26,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:26,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:26,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:26,848 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/51ca97b53d754dc9bcb928f83a4e84de as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/51ca97b53d754dc9bcb928f83a4e84de 2024-11-24T08:14:26,852 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/11b086ce04324f58b08b7421c5bed46f 2024-11-24T08:14:26,855 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/C of 0e4dbe9176ee2b005eb8db52027271c6 into 51ca97b53d754dc9bcb928f83a4e84de(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:26,855 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:26,855 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/C, priority=13, startTime=1732436066187; duration=0sec 2024-11-24T08:14:26,855 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:26,855 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:C 2024-11-24T08:14:26,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/38ad01f74e0b42a8ad2ae1e7fd9a3928 is 50, key is test_row_0/B:col10/1732436066275/Put/seqid=0 2024-11-24T08:14:26,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741998_1174 (size=12151) 2024-11-24T08:14:26,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436126920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436126949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436126950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436126950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436126950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:26,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T08:14:26,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:26,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:26,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:26,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:26,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:26,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T08:14:27,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T08:14:27,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:27,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:27,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:27,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:27,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:27,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/38ad01f74e0b42a8ad2ae1e7fd9a3928 2024-11-24T08:14:27,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/51d984892b0b46a7a81dbd8d8d9b6ca7 is 50, key is test_row_0/C:col10/1732436066275/Put/seqid=0 2024-11-24T08:14:27,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T08:14:27,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:27,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:27,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:27,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:27,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:27,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741999_1175 (size=12151) 2024-11-24T08:14:27,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/51d984892b0b46a7a81dbd8d8d9b6ca7 2024-11-24T08:14:27,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/11b086ce04324f58b08b7421c5bed46f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/11b086ce04324f58b08b7421c5bed46f 2024-11-24T08:14:27,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/11b086ce04324f58b08b7421c5bed46f, entries=250, sequenceid=219, filesize=47.3 K 2024-11-24T08:14:27,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/38ad01f74e0b42a8ad2ae1e7fd9a3928 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/38ad01f74e0b42a8ad2ae1e7fd9a3928 2024-11-24T08:14:27,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/38ad01f74e0b42a8ad2ae1e7fd9a3928, entries=150, sequenceid=219, filesize=11.9 K 2024-11-24T08:14:27,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/51d984892b0b46a7a81dbd8d8d9b6ca7 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/51d984892b0b46a7a81dbd8d8d9b6ca7 2024-11-24T08:14:27,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/51d984892b0b46a7a81dbd8d8d9b6ca7, entries=150, sequenceid=219, filesize=11.9 K 2024-11-24T08:14:27,310 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 0e4dbe9176ee2b005eb8db52027271c6 in 1033ms, sequenceid=219, compaction requested=false 2024-11-24T08:14:27,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:27,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-24T08:14:27,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:27,432 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:14:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:27,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:27,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244e349211cc1444f4984299b81758e621_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436066292/Put/seqid=0 2024-11-24T08:14:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:27,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:27,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742000_1176 (size=12304) 2024-11-24T08:14:27,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:27,453 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244e349211cc1444f4984299b81758e621_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244e349211cc1444f4984299b81758e621_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:27,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/323afd8881d4413d86e9220210389605, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:27,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/323afd8881d4413d86e9220210389605 is 175, key is test_row_0/A:col10/1732436066292/Put/seqid=0 2024-11-24T08:14:27,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742001_1177 (size=31105) 2024-11-24T08:14:27,461 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/323afd8881d4413d86e9220210389605 2024-11-24T08:14:27,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/ad678ca5c818406787fc31c15399fcc9 is 50, key is test_row_0/B:col10/1732436066292/Put/seqid=0 2024-11-24T08:14:27,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742002_1178 (size=12151) 2024-11-24T08:14:27,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436127479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732436067472, max timestamp=1732436067482Num uniques 859; 2024-11-24T08:14:27,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T08:14:27,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732436067472, max timestamp=1732436067482Num uniques 859; 2024-11-24T08:14:27,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T08:14:27,487 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/ad678ca5c818406787fc31c15399fcc9 2024-11-24T08:14:27,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/0be0e95fd37c4f3588266637461826ed is 50, key is test_row_0/C:col10/1732436066292/Put/seqid=0 2024-11-24T08:14:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T08:14:27,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436127503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742003_1179 (size=12151) 2024-11-24T08:14:27,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436127504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436127505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436127505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436127589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436127620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436127620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,621 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436127620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436127621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436127810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436127830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436127830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436127830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:27,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436127831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:27,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,835 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:27,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:27,913 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/0be0e95fd37c4f3588266637461826ed 2024-11-24T08:14:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/323afd8881d4413d86e9220210389605 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/323afd8881d4413d86e9220210389605 2024-11-24T08:14:27,925 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/323afd8881d4413d86e9220210389605, entries=150, sequenceid=232, filesize=30.4 K 2024-11-24T08:14:27,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/ad678ca5c818406787fc31c15399fcc9 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ad678ca5c818406787fc31c15399fcc9 2024-11-24T08:14:27,934 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ad678ca5c818406787fc31c15399fcc9, entries=150, sequenceid=232, filesize=11.9 K 2024-11-24T08:14:27,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/0be0e95fd37c4f3588266637461826ed as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/0be0e95fd37c4f3588266637461826ed 2024-11-24T08:14:27,941 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/0be0e95fd37c4f3588266637461826ed, entries=150, sequenceid=232, filesize=11.9 K 2024-11-24T08:14:27,942 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 0e4dbe9176ee2b005eb8db52027271c6 in 510ms, sequenceid=232, compaction requested=true 2024-11-24T08:14:27,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:27,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:27,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-24T08:14:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-24T08:14:27,946 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-24T08:14:27,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5860 sec 2024-11-24T08:14:27,950 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.5950 sec 2024-11-24T08:14:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:28,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=147.60 KB heapSize=423.55 KB 2024-11-24T08:14:28,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:28,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:28,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:28,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:28,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:28,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:28,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241e595595237f421ea1db0aa8ad79fa77_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436067503/Put/seqid=0 2024-11-24T08:14:28,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436128140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436128144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436128147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,151 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436128149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742004_1180 (size=14894) 2024-11-24T08:14:28,160 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436128151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,165 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241e595595237f421ea1db0aa8ad79fa77_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241e595595237f421ea1db0aa8ad79fa77_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:28,166 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/b958897b2ef14534831e0badc8740db2, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:28,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/b958897b2ef14534831e0badc8740db2 is 175, key is test_row_0/A:col10/1732436067503/Put/seqid=0 2024-11-24T08:14:28,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742005_1181 (size=39849) 2024-11-24T08:14:28,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436128260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436128260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436128261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436128270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436128470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436128470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436128470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436128481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-24T08:14:28,510 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:28,511 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-24T08:14:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T08:14:28,513 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:28,514 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:28,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:28,575 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/b958897b2ef14534831e0badc8740db2 2024-11-24T08:14:28,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/89b809dc767f4c9e81b7fabd9f2925f7 is 50, key is test_row_0/B:col10/1732436067503/Put/seqid=0 2024-11-24T08:14:28,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742006_1182 (size=12251) 2024-11-24T08:14:28,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/89b809dc767f4c9e81b7fabd9f2925f7 2024-11-24T08:14:28,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/238d4aa162644e24b94fb763282cff7d is 50, key is test_row_0/C:col10/1732436067503/Put/seqid=0 2024-11-24T08:14:28,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742007_1183 (size=12251) 2024-11-24T08:14:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T08:14:28,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/238d4aa162644e24b94fb763282cff7d 2024-11-24T08:14:28,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/b958897b2ef14534831e0badc8740db2 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/b958897b2ef14534831e0badc8740db2 2024-11-24T08:14:28,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/b958897b2ef14534831e0badc8740db2, entries=200, sequenceid=265, filesize=38.9 K 2024-11-24T08:14:28,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/89b809dc767f4c9e81b7fabd9f2925f7 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/89b809dc767f4c9e81b7fabd9f2925f7 2024-11-24T08:14:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/89b809dc767f4c9e81b7fabd9f2925f7, entries=150, sequenceid=265, filesize=12.0 K 2024-11-24T08:14:28,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/238d4aa162644e24b94fb763282cff7d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/238d4aa162644e24b94fb763282cff7d 2024-11-24T08:14:28,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/238d4aa162644e24b94fb763282cff7d, entries=150, sequenceid=265, filesize=12.0 K 2024-11-24T08:14:28,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~423.50 KB/433664, currentSize=40.25 KB/41220 for 0e4dbe9176ee2b005eb8db52027271c6 in 519ms, sequenceid=265, compaction requested=true 2024-11-24T08:14:28,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:28,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:28,643 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:14:28,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:28,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:28,643 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:14:28,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:28,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,646 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49080 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:14:28,646 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150824 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:14:28,646 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/A is initiating minor compaction (all files) 2024-11-24T08:14:28,646 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/B is initiating minor compaction (all files) 2024-11-24T08:14:28,646 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/B in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:28,646 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/A in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:28,646 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/adc3e825e3034cda81ac6b260c6b68a1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/38ad01f74e0b42a8ad2ae1e7fd9a3928, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ad678ca5c818406787fc31c15399fcc9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/89b809dc767f4c9e81b7fabd9f2925f7] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=47.9 K 2024-11-24T08:14:28,646 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/35ea893af27e4b65933065beb710179f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/11b086ce04324f58b08b7421c5bed46f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/323afd8881d4413d86e9220210389605, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/b958897b2ef14534831e0badc8740db2] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=147.3 K 2024-11-24T08:14:28,646 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:28,646 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/35ea893af27e4b65933065beb710179f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/11b086ce04324f58b08b7421c5bed46f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/323afd8881d4413d86e9220210389605, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/b958897b2ef14534831e0badc8740db2] 2024-11-24T08:14:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,646 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting adc3e825e3034cda81ac6b260c6b68a1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732436064905 2024-11-24T08:14:28,647 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35ea893af27e4b65933065beb710179f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732436064905 2024-11-24T08:14:28,647 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 38ad01f74e0b42a8ad2ae1e7fd9a3928, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732436065625 2024-11-24T08:14:28,647 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11b086ce04324f58b08b7421c5bed46f, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732436065574 2024-11-24T08:14:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,647 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting ad678ca5c818406787fc31c15399fcc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732436066284 2024-11-24T08:14:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,648 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 89b809dc767f4c9e81b7fabd9f2925f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732436067488 2024-11-24T08:14:28,648 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 323afd8881d4413d86e9220210389605, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732436066284 2024-11-24T08:14:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,649 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b958897b2ef14534831e0badc8740db2, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732436067483 2024-11-24T08:14:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-24T08:14:28,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:28,667 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-24T08:14:28,667 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#B#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:28,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:28,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:28,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:28,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:28,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:28,668 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/69f713f3c4f1496c8fc5dbae15563960 is 50, key is test_row_0/B:col10/1732436067503/Put/seqid=0 2024-11-24T08:14:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,669 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:28,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,689 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124a551a56bb5f445e0b16ec244d9d6e6c1_0e4dbe9176ee2b005eb8db52027271c6 store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,692 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124a551a56bb5f445e0b16ec244d9d6e6c1_0e4dbe9176ee2b005eb8db52027271c6, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,692 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a551a56bb5f445e0b16ec244d9d6e6c1_0e4dbe9176ee2b005eb8db52027271c6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742008_1184 (size=12763) 2024-11-24T08:14:28,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,706 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/69f713f3c4f1496c8fc5dbae15563960 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/69f713f3c4f1496c8fc5dbae15563960 2024-11-24T08:14:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112466c6481eb1bc4876a372e2dcdbcc1e4a_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_1/A:col10/1732436068141/Put/seqid=0 2024-11-24T08:14:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,714 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/B of 0e4dbe9176ee2b005eb8db52027271c6 into 69f713f3c4f1496c8fc5dbae15563960(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:28,714 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:28,714 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/B, priority=12, startTime=1732436068643; duration=0sec 2024-11-24T08:14:28,714 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:28,715 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:B 2024-11-24T08:14:28,715 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:14:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,717 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49080 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:14:28,717 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/C is initiating minor compaction (all files) 2024-11-24T08:14:28,717 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/C in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:28,717 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/51ca97b53d754dc9bcb928f83a4e84de, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/51d984892b0b46a7a81dbd8d8d9b6ca7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/0be0e95fd37c4f3588266637461826ed, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/238d4aa162644e24b94fb763282cff7d] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=47.9 K 2024-11-24T08:14:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,717 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 51ca97b53d754dc9bcb928f83a4e84de, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732436064905 2024-11-24T08:14:28,718 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 51d984892b0b46a7a81dbd8d8d9b6ca7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732436065625 2024-11-24T08:14:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,718 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0be0e95fd37c4f3588266637461826ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732436066284 2024-11-24T08:14:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,719 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 238d4aa162644e24b94fb763282cff7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732436067488 2024-11-24T08:14:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742009_1185 (size=4469) 2024-11-24T08:14:28,740 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#A#compaction#154 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:28,741 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/caa500ffbb77440691f5871f3bf0099b is 175, key is test_row_0/A:col10/1732436067503/Put/seqid=0 2024-11-24T08:14:28,748 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#C#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:28,748 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/1d96912b895e498e8f5a3d88f06e9d20 is 50, key is test_row_0/C:col10/1732436067503/Put/seqid=0 2024-11-24T08:14:28,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742011_1187 (size=31717) 2024-11-24T08:14:28,770 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/caa500ffbb77440691f5871f3bf0099b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/caa500ffbb77440691f5871f3bf0099b 2024-11-24T08:14:28,776 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/A of 0e4dbe9176ee2b005eb8db52027271c6 into caa500ffbb77440691f5871f3bf0099b(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:28,776 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:28,776 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/A, priority=12, startTime=1732436068643; duration=0sec 2024-11-24T08:14:28,776 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:28,776 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:A 2024-11-24T08:14:28,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436068764, max timestamp=1732436068786Num uniques 859; 2024-11-24T08:14:28,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:28,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436068764, max timestamp=1732436068786Num uniques 859; 2024-11-24T08:14:28,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:28,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742010_1186 (size=9914) 2024-11-24T08:14:28,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436068764, max timestamp=1732436068786Num uniques 859; 2024-11-24T08:14:28,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:28,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:28,801 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112466c6481eb1bc4876a372e2dcdbcc1e4a_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112466c6481eb1bc4876a372e2dcdbcc1e4a_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:28,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742012_1188 (size=12763) 2024-11-24T08:14:28,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/761cf8fd3f9b434399d60d6532748bb0, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:28,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/761cf8fd3f9b434399d60d6532748bb0 is 175, key is test_row_1/A:col10/1732436068141/Put/seqid=0 2024-11-24T08:14:28,815 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/1d96912b895e498e8f5a3d88f06e9d20 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/1d96912b895e498e8f5a3d88f06e9d20 2024-11-24T08:14:28,821 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/C of 0e4dbe9176ee2b005eb8db52027271c6 into 1d96912b895e498e8f5a3d88f06e9d20(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:28,822 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:28,822 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/C, priority=12, startTime=1732436068643; duration=0sec 2024-11-24T08:14:28,822 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:28,822 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:C 2024-11-24T08:14:28,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742013_1189 (size=22561) 2024-11-24T08:14:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T08:14:28,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436128826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732436068810, max timestamp=1732436068828Num uniques 1018; 2024-11-24T08:14:28,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732436068810, max timestamp=1732436068828Num uniques 1018; 2024-11-24T08:14:28,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-24T08:14:28,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-24T08:14:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732436068810, max timestamp=1732436068828Num uniques 1018; 2024-11-24T08:14:28,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-24T08:14:28,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436128859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436128861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436128863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436128863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436128940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436128971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436128971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436128980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436128980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:28,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:29,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T08:14:29,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436129150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:29,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436129180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:29,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436129181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:29,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436129190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:29,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436129190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:29,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:29,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:29,227 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/761cf8fd3f9b434399d60d6532748bb0 2024-11-24T08:14:29,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/ebe202bcba464ee89b05aaa8839ecc1e is 50, key is test_row_1/B:col10/1732436068141/Put/seqid=0 2024-11-24T08:14:29,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742014_1190 (size=9857) 2024-11-24T08:14:29,254 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/ebe202bcba464ee89b05aaa8839ecc1e 2024-11-24T08:14:29,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/362f62cc63794aa69fdd20875c6894db is 50, key is test_row_1/C:col10/1732436068141/Put/seqid=0 2024-11-24T08:14:29,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742015_1191 (size=9857) 2024-11-24T08:14:29,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436129460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:29,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:29,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436129500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:29,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436129501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:29,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436129501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:29,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436129501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:29,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T08:14:29,693 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/362f62cc63794aa69fdd20875c6894db 2024-11-24T08:14:29,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/761cf8fd3f9b434399d60d6532748bb0 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/761cf8fd3f9b434399d60d6532748bb0 2024-11-24T08:14:29,704 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/761cf8fd3f9b434399d60d6532748bb0, entries=100, sequenceid=274, filesize=22.0 K 2024-11-24T08:14:29,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/ebe202bcba464ee89b05aaa8839ecc1e as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ebe202bcba464ee89b05aaa8839ecc1e 2024-11-24T08:14:29,711 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ebe202bcba464ee89b05aaa8839ecc1e, entries=100, sequenceid=274, filesize=9.6 K 2024-11-24T08:14:29,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/362f62cc63794aa69fdd20875c6894db as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/362f62cc63794aa69fdd20875c6894db 2024-11-24T08:14:29,718 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/362f62cc63794aa69fdd20875c6894db, entries=100, sequenceid=274, filesize=9.6 K 2024-11-24T08:14:29,719 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=154.31 KB/158010 for 0e4dbe9176ee2b005eb8db52027271c6 in 1052ms, sequenceid=274, compaction requested=false 2024-11-24T08:14:29,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:29,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:29,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-24T08:14:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-24T08:14:29,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-24T08:14:29,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2070 sec 2024-11-24T08:14:29,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.2130 sec 2024-11-24T08:14:29,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:29,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=161.02 KB heapSize=450.47 KB 2024-11-24T08:14:29,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:29,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T08:14:29,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:29,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T08:14:29,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:29,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T08:14:29,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b4c02828c5bb4bd8bca4d5919bde377f_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436069973/Put/seqid=0 2024-11-24T08:14:30,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742016_1192 (size=14994) 2024-11-24T08:14:30,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436130010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:30,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:30,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436130011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:30,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:30,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436130012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:30,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436130014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:30,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436130014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,024 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:30,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436130119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:30,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436130130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:30,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436130130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:30,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:30,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436130329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:30,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:30,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436130340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:30,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436130340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:30,407 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:30,414 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b4c02828c5bb4bd8bca4d5919bde377f_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b4c02828c5bb4bd8bca4d5919bde377f_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:30,417 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/e285f5867731477ca91a83af914f4850, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:30,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/e285f5867731477ca91a83af914f4850 is 175, key is test_row_0/A:col10/1732436069973/Put/seqid=0 2024-11-24T08:14:30,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742017_1193 (size=39949) 2024-11-24T08:14:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436130640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:30,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436130650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:30,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436130650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:30,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-24T08:14:30,661 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:30,662 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-24T08:14:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T08:14:30,664 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:30,665 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:30,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:30,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T08:14:30,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T08:14:30,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:30,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:30,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:30,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:30,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:30,833 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=332, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/e285f5867731477ca91a83af914f4850 2024-11-24T08:14:30,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/c74f7facd6a54fa4871259819b279dad is 50, key is test_row_0/B:col10/1732436069973/Put/seqid=0 2024-11-24T08:14:30,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742018_1194 (size=12301) 2024-11-24T08:14:30,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/c74f7facd6a54fa4871259819b279dad 2024-11-24T08:14:30,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/612912e606614bf592562ddcc8997201 is 50, key is test_row_0/C:col10/1732436069973/Put/seqid=0 2024-11-24T08:14:30,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742019_1195 (size=12301) 2024-11-24T08:14:30,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T08:14:30,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:30,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:30,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:30,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:30,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:30,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:30,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T08:14:31,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436131019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:31,024 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_0' in 0e4dbe9176ee2b005eb8db52027271c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:14:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:31,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:31,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:31,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436131031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:31,035 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 0e4dbe9176ee2b005eb8db52027271c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:14:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:31,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:31,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T08:14:31,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:31,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:31,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:31,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:31,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:31,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:31,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:31,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436131156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:31,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:31,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:31,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:31,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436131169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:31,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:31,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436131170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:31,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:31,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/612912e606614bf592562ddcc8997201 2024-11-24T08:14:31,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/e285f5867731477ca91a83af914f4850 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/e285f5867731477ca91a83af914f4850 2024-11-24T08:14:31,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T08:14:31,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:31,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:31,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:31,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:31,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:31,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:31,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/e285f5867731477ca91a83af914f4850, entries=200, sequenceid=332, filesize=39.0 K 2024-11-24T08:14:31,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/c74f7facd6a54fa4871259819b279dad as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/c74f7facd6a54fa4871259819b279dad 2024-11-24T08:14:31,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T08:14:31,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/c74f7facd6a54fa4871259819b279dad, entries=150, sequenceid=332, filesize=12.0 K 2024-11-24T08:14:31,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/612912e606614bf592562ddcc8997201 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/612912e606614bf592562ddcc8997201 2024-11-24T08:14:31,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/612912e606614bf592562ddcc8997201, entries=150, sequenceid=332, filesize=12.0 K 2024-11-24T08:14:31,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~450.42 KB/461232, currentSize=33.54 KB/34350 for 0e4dbe9176ee2b005eb8db52027271c6 in 1324ms, sequenceid=332, compaction requested=true 2024-11-24T08:14:31,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:31,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:31,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:31,304 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:31,304 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:31,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:31,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:31,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:31,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:31,305 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94227 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:31,305 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/A is initiating minor compaction (all files) 2024-11-24T08:14:31,306 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/A in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:31,306 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/caa500ffbb77440691f5871f3bf0099b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/761cf8fd3f9b434399d60d6532748bb0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/e285f5867731477ca91a83af914f4850] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=92.0 K 2024-11-24T08:14:31,306 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:31,306 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/caa500ffbb77440691f5871f3bf0099b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/761cf8fd3f9b434399d60d6532748bb0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/e285f5867731477ca91a83af914f4850] 2024-11-24T08:14:31,306 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34921 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:31,306 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/B is initiating minor compaction (all files) 2024-11-24T08:14:31,306 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/B in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:31,306 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/69f713f3c4f1496c8fc5dbae15563960, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ebe202bcba464ee89b05aaa8839ecc1e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/c74f7facd6a54fa4871259819b279dad] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=34.1 K 2024-11-24T08:14:31,308 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting caa500ffbb77440691f5871f3bf0099b, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732436067488 2024-11-24T08:14:31,308 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 69f713f3c4f1496c8fc5dbae15563960, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732436067488 2024-11-24T08:14:31,309 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 761cf8fd3f9b434399d60d6532748bb0, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732436068141 2024-11-24T08:14:31,309 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting ebe202bcba464ee89b05aaa8839ecc1e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732436068141 2024-11-24T08:14:31,310 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e285f5867731477ca91a83af914f4850, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732436068859 2024-11-24T08:14:31,310 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting c74f7facd6a54fa4871259819b279dad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732436068859 2024-11-24T08:14:31,319 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:31,322 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#B#compaction#163 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:31,323 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/aec89cec8ee94211ae87bd6dd19ee9ad is 50, key is test_row_0/B:col10/1732436069973/Put/seqid=0 2024-11-24T08:14:31,324 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411242c2ffad3be804eaeb684ea9a4d4c459f_0e4dbe9176ee2b005eb8db52027271c6 store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:31,326 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411242c2ffad3be804eaeb684ea9a4d4c459f_0e4dbe9176ee2b005eb8db52027271c6, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:31,326 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242c2ffad3be804eaeb684ea9a4d4c459f_0e4dbe9176ee2b005eb8db52027271c6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:31,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742021_1197 (size=4469) 2024-11-24T08:14:31,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742020_1196 (size=12915) 2024-11-24T08:14:31,339 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#A#compaction#162 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:31,340 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/81de9901a1f240a79aa8fe662165fa81 is 175, key is test_row_0/A:col10/1732436069973/Put/seqid=0 2024-11-24T08:14:31,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742022_1198 (size=31869) 2024-11-24T08:14:31,348 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/aec89cec8ee94211ae87bd6dd19ee9ad as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/aec89cec8ee94211ae87bd6dd19ee9ad 2024-11-24T08:14:31,353 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/B of 0e4dbe9176ee2b005eb8db52027271c6 into aec89cec8ee94211ae87bd6dd19ee9ad(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:31,353 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:31,353 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/B, priority=13, startTime=1732436071304; duration=0sec 2024-11-24T08:14:31,353 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:31,353 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:B 2024-11-24T08:14:31,353 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:31,354 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34921 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:31,354 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/C is initiating minor compaction (all files) 2024-11-24T08:14:31,354 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/C in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:31,354 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/1d96912b895e498e8f5a3d88f06e9d20, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/362f62cc63794aa69fdd20875c6894db, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/612912e606614bf592562ddcc8997201] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=34.1 K 2024-11-24T08:14:31,355 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d96912b895e498e8f5a3d88f06e9d20, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1732436067488 2024-11-24T08:14:31,355 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 362f62cc63794aa69fdd20875c6894db, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732436068141 2024-11-24T08:14:31,356 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 612912e606614bf592562ddcc8997201, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732436068859 2024-11-24T08:14:31,363 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#C#compaction#164 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:31,364 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/19b309490d6a4efe9432b3d226cff92e is 50, key is test_row_0/C:col10/1732436069973/Put/seqid=0 2024-11-24T08:14:31,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742023_1199 (size=12915) 2024-11-24T08:14:31,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-24T08:14:31,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:31,432 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T08:14:31,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:31,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:31,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:31,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:31,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:31,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:31,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f5e0fb20ef944fb396398a7d05d0bed0_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436070010/Put/seqid=0 2024-11-24T08:14:31,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742024_1200 (size=12454) 2024-11-24T08:14:31,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,474 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f5e0fb20ef944fb396398a7d05d0bed0_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f5e0fb20ef944fb396398a7d05d0bed0_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:31,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/a2e9baa39a2548678a109af274e63b0d, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:31,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/a2e9baa39a2548678a109af274e63b0d is 175, key is test_row_0/A:col10/1732436070010/Put/seqid=0 2024-11-24T08:14:31,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742025_1201 (size=31255) 2024-11-24T08:14:31,481 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/a2e9baa39a2548678a109af274e63b0d 2024-11-24T08:14:31,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/9ef4dd0631d5495c94fd30085d8a9a45 is 50, key is test_row_0/B:col10/1732436070010/Put/seqid=0 2024-11-24T08:14:31,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742026_1202 (size=12301) 2024-11-24T08:14:31,498 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/9ef4dd0631d5495c94fd30085d8a9a45 2024-11-24T08:14:31,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/f7147b27a96842dc88b27aabfc6685a9 is 50, key is test_row_0/C:col10/1732436070010/Put/seqid=0 2024-11-24T08:14:31,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742027_1203 (size=12301) 2024-11-24T08:14:31,754 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/81de9901a1f240a79aa8fe662165fa81 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/81de9901a1f240a79aa8fe662165fa81 2024-11-24T08:14:31,759 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/A of 0e4dbe9176ee2b005eb8db52027271c6 into 81de9901a1f240a79aa8fe662165fa81(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:31,759 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:31,759 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/A, priority=13, startTime=1732436071304; duration=0sec 2024-11-24T08:14:31,760 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:31,760 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:A 2024-11-24T08:14:31,773 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/19b309490d6a4efe9432b3d226cff92e as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/19b309490d6a4efe9432b3d226cff92e 2024-11-24T08:14:31,779 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/C of 0e4dbe9176ee2b005eb8db52027271c6 into 19b309490d6a4efe9432b3d226cff92e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:31,779 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:31,779 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/C, priority=13, startTime=1732436071305; duration=0sec 2024-11-24T08:14:31,779 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:31,779 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:C 2024-11-24T08:14:31,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T08:14:31,957 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/f7147b27a96842dc88b27aabfc6685a9 2024-11-24T08:14:31,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/a2e9baa39a2548678a109af274e63b0d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/a2e9baa39a2548678a109af274e63b0d 2024-11-24T08:14:31,967 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/a2e9baa39a2548678a109af274e63b0d, entries=150, sequenceid=341, filesize=30.5 K 2024-11-24T08:14:31,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/9ef4dd0631d5495c94fd30085d8a9a45 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/9ef4dd0631d5495c94fd30085d8a9a45 2024-11-24T08:14:31,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,973 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/9ef4dd0631d5495c94fd30085d8a9a45, entries=150, sequenceid=341, filesize=12.0 K 2024-11-24T08:14:31,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/f7147b27a96842dc88b27aabfc6685a9 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/f7147b27a96842dc88b27aabfc6685a9 2024-11-24T08:14:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,984 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/f7147b27a96842dc88b27aabfc6685a9, entries=150, sequenceid=341, filesize=12.0 K 2024-11-24T08:14:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,985 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0e4dbe9176ee2b005eb8db52027271c6 in 553ms, sequenceid=341, compaction requested=false 2024-11-24T08:14:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:31,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:31,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-24T08:14:31,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-24T08:14:31,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-24T08:14:31,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3220 sec 2024-11-24T08:14:31,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.3280 sec 2024-11-24T08:14:31,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:32,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:14:32,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:32,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:32,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:32,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:32,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:32,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d29384ae0f1b41fbbe524160686242ee_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436072252/Put/seqid=0 2024-11-24T08:14:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742028_1204 (size=22618) 2024-11-24T08:14:32,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732436072287, max timestamp=1732436072317Num uniques 859; 2024-11-24T08:14:32,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T08:14:32,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732436072287, max timestamp=1732436072317Num uniques 859; 2024-11-24T08:14:32,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T08:14:32,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732436072287, max timestamp=1732436072317Num uniques 859; 2024-11-24T08:14:32,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T08:14:32,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436132357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:32,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:32,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436132368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:32,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:32,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436132371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:32,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:32,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436132480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:32,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436132481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:32,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:32,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436132481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:32,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:32,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436132689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:32,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,693 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:32,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:32,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436132690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:32,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:32,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:32,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436132689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:32,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:32,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:32,717 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:32,725 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d29384ae0f1b41fbbe524160686242ee_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d29384ae0f1b41fbbe524160686242ee_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:32,727 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/bb26dcf7e5bd44dea47f6c6cf2337635, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:32,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/bb26dcf7e5bd44dea47f6c6cf2337635 is 175, key is test_row_0/A:col10/1732436072252/Put/seqid=0 2024-11-24T08:14:32,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742029_1205 (size=66023) 2024-11-24T08:14:32,740 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/bb26dcf7e5bd44dea47f6c6cf2337635 2024-11-24T08:14:32,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/0dafce8c8d244d62906d665443e30853 is 50, key is test_row_0/B:col10/1732436072252/Put/seqid=0 2024-11-24T08:14:32,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742030_1206 (size=12301) 2024-11-24T08:14:32,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-24T08:14:32,820 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:32,821 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:32,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-24T08:14:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T08:14:32,823 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:32,823 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:32,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:32,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T08:14:32,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-24T08:14:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:32,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:33,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436133000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:33,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436133000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:33,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,004 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:33,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:33,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436133000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:33,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:33,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436133039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:33,043 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_0' in 0e4dbe9176ee2b005eb8db52027271c6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:14:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,044 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:33,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436133060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:33,064 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 0e4dbe9176ee2b005eb8db52027271c6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:14:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:33,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-24T08:14:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:33,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:33,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:33,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T08:14:33,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/0dafce8c8d244d62906d665443e30853 2024-11-24T08:14:33,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/3879ed8611744658919af71af67c2a22 is 50, key is test_row_0/C:col10/1732436072252/Put/seqid=0 2024-11-24T08:14:33,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742031_1207 (size=12301) 2024-11-24T08:14:33,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/3879ed8611744658919af71af67c2a22 2024-11-24T08:14:33,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/bb26dcf7e5bd44dea47f6c6cf2337635 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/bb26dcf7e5bd44dea47f6c6cf2337635 2024-11-24T08:14:33,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/bb26dcf7e5bd44dea47f6c6cf2337635, entries=350, sequenceid=353, filesize=64.5 K 2024-11-24T08:14:33,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/0dafce8c8d244d62906d665443e30853 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/0dafce8c8d244d62906d665443e30853 2024-11-24T08:14:33,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/0dafce8c8d244d62906d665443e30853, entries=150, sequenceid=353, filesize=12.0 K 2024-11-24T08:14:33,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/3879ed8611744658919af71af67c2a22 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/3879ed8611744658919af71af67c2a22 2024-11-24T08:14:33,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-24T08:14:33,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/3879ed8611744658919af71af67c2a22, entries=150, sequenceid=353, filesize=12.0 K 2024-11-24T08:14:33,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:33,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:33,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:33,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:33,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:33,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 0e4dbe9176ee2b005eb8db52027271c6 in 1025ms, sequenceid=353, compaction requested=true 2024-11-24T08:14:33,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:33,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:33,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:33,285 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:33,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:33,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:33,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:33,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:33,285 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T08:14:33,285 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:33,287 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129147 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:33,287 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/A is initiating minor compaction (all files) 2024-11-24T08:14:33,287 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/A in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:33,287 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/81de9901a1f240a79aa8fe662165fa81, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/a2e9baa39a2548678a109af274e63b0d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/bb26dcf7e5bd44dea47f6c6cf2337635] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=126.1 K 2024-11-24T08:14:33,288 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:33,288 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/81de9901a1f240a79aa8fe662165fa81, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/a2e9baa39a2548678a109af274e63b0d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/bb26dcf7e5bd44dea47f6c6cf2337635] 2024-11-24T08:14:33,288 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:33,288 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/B is initiating minor compaction (all files) 2024-11-24T08:14:33,288 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/B in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:33,288 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/aec89cec8ee94211ae87bd6dd19ee9ad, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/9ef4dd0631d5495c94fd30085d8a9a45, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/0dafce8c8d244d62906d665443e30853] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=36.6 K 2024-11-24T08:14:33,288 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 81de9901a1f240a79aa8fe662165fa81, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732436068859 2024-11-24T08:14:33,289 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting aec89cec8ee94211ae87bd6dd19ee9ad, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732436068859 2024-11-24T08:14:33,289 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ef4dd0631d5495c94fd30085d8a9a45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732436070009 2024-11-24T08:14:33,290 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e9baa39a2548678a109af274e63b0d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732436070009 2024-11-24T08:14:33,290 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dafce8c8d244d62906d665443e30853, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732436072245 2024-11-24T08:14:33,290 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting bb26dcf7e5bd44dea47f6c6cf2337635, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732436072196 2024-11-24T08:14:33,306 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:33,307 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#B#compaction#171 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:33,308 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/e7a08f8a15f84fb6bb42b3d0ea8c1310 is 50, key is test_row_0/B:col10/1732436072252/Put/seqid=0 2024-11-24T08:14:33,308 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124c9a5255e97714ed6a002239c48fa2727_0e4dbe9176ee2b005eb8db52027271c6 store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:33,311 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124c9a5255e97714ed6a002239c48fa2727_0e4dbe9176ee2b005eb8db52027271c6, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:33,311 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124c9a5255e97714ed6a002239c48fa2727_0e4dbe9176ee2b005eb8db52027271c6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:33,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742032_1208 (size=13017) 2024-11-24T08:14:33,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742033_1209 (size=4469) 2024-11-24T08:14:33,363 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/e7a08f8a15f84fb6bb42b3d0ea8c1310 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/e7a08f8a15f84fb6bb42b3d0ea8c1310 2024-11-24T08:14:33,366 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#A#compaction#172 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:33,366 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/7c45cb6cc6ac415d94f63aa1acbccc23 is 175, key is test_row_0/A:col10/1732436072252/Put/seqid=0 2024-11-24T08:14:33,374 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/B of 0e4dbe9176ee2b005eb8db52027271c6 into e7a08f8a15f84fb6bb42b3d0ea8c1310(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:33,374 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:33,374 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/B, priority=13, startTime=1732436073285; duration=0sec 2024-11-24T08:14:33,374 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:33,374 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:B 2024-11-24T08:14:33,374 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:33,376 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:33,376 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/C is initiating minor compaction (all files) 2024-11-24T08:14:33,376 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/C in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:33,376 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/19b309490d6a4efe9432b3d226cff92e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/f7147b27a96842dc88b27aabfc6685a9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/3879ed8611744658919af71af67c2a22] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=36.6 K 2024-11-24T08:14:33,377 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19b309490d6a4efe9432b3d226cff92e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732436068859 2024-11-24T08:14:33,377 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7147b27a96842dc88b27aabfc6685a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1732436070009 2024-11-24T08:14:33,377 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3879ed8611744658919af71af67c2a22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732436072245 2024-11-24T08:14:33,406 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#C#compaction#173 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:33,406 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/8282676917b743a98f42332c9735ea78 is 50, key is test_row_0/C:col10/1732436072252/Put/seqid=0 2024-11-24T08:14:33,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742034_1210 (size=31971) 2024-11-24T08:14:33,420 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/7c45cb6cc6ac415d94f63aa1acbccc23 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/7c45cb6cc6ac415d94f63aa1acbccc23 2024-11-24T08:14:33,427 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/A of 0e4dbe9176ee2b005eb8db52027271c6 into 7c45cb6cc6ac415d94f63aa1acbccc23(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:33,427 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:33,427 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/A, priority=13, startTime=1732436073285; duration=0sec 2024-11-24T08:14:33,427 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:33,427 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:A 2024-11-24T08:14:33,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-24T08:14:33,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:33,437 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-24T08:14:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:33,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:33,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T08:14:33,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742035_1211 (size=13017) 2024-11-24T08:14:33,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243c1da917b08b48118aac8c3ae28b0799_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436072366/Put/seqid=0 2024-11-24T08:14:33,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742036_1212 (size=12454) 2024-11-24T08:14:33,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:33,498 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243c1da917b08b48118aac8c3ae28b0799_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243c1da917b08b48118aac8c3ae28b0799_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:33,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/424b85d251ff4484a4171f5bb520925c, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:33,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/424b85d251ff4484a4171f5bb520925c is 175, key is test_row_0/A:col10/1732436072366/Put/seqid=0 2024-11-24T08:14:33,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:33,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:33,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:33,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436133519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:33,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:33,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742037_1213 (size=31255) 2024-11-24T08:14:33,526 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=391, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/424b85d251ff4484a4171f5bb520925c 2024-11-24T08:14:33,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436133521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:33,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436133523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:33,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/cb42c2b09aaa4c4fa19f51217c4e9f18 is 50, key is test_row_0/B:col10/1732436072366/Put/seqid=0 2024-11-24T08:14:33,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742038_1214 (size=12301) 2024-11-24T08:14:33,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:33,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436133629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,641 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:33,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:33,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436133640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:33,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436133640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:33,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436133850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:33,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:33,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436133850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:33,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436133860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:33,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:33,875 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/8282676917b743a98f42332c9735ea78 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/8282676917b743a98f42332c9735ea78 2024-11-24T08:14:33,883 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/C of 0e4dbe9176ee2b005eb8db52027271c6 into 8282676917b743a98f42332c9735ea78(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:33,883 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:33,883 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/C, priority=13, startTime=1732436073285; duration=0sec 2024-11-24T08:14:33,883 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:33,883 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:C 2024-11-24T08:14:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T08:14:33,979 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/cb42c2b09aaa4c4fa19f51217c4e9f18 2024-11-24T08:14:33,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/ae9a1faa1bd7472cae6108aaf8c7995a is 50, key is test_row_0/C:col10/1732436072366/Put/seqid=0 2024-11-24T08:14:34,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742039_1215 (size=12301) 2024-11-24T08:14:34,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:34,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436134160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:34,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:34,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436134161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:34,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:34,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:34,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:34,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436134180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:34,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:34,447 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/ae9a1faa1bd7472cae6108aaf8c7995a 2024-11-24T08:14:34,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/424b85d251ff4484a4171f5bb520925c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/424b85d251ff4484a4171f5bb520925c 2024-11-24T08:14:34,460 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/424b85d251ff4484a4171f5bb520925c, entries=150, sequenceid=391, filesize=30.5 K 2024-11-24T08:14:34,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/cb42c2b09aaa4c4fa19f51217c4e9f18 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/cb42c2b09aaa4c4fa19f51217c4e9f18 2024-11-24T08:14:34,466 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/cb42c2b09aaa4c4fa19f51217c4e9f18, entries=150, sequenceid=391, filesize=12.0 K 2024-11-24T08:14:34,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/ae9a1faa1bd7472cae6108aaf8c7995a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/ae9a1faa1bd7472cae6108aaf8c7995a 2024-11-24T08:14:34,473 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/ae9a1faa1bd7472cae6108aaf8c7995a, entries=150, sequenceid=391, filesize=12.0 K 2024-11-24T08:14:34,474 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 0e4dbe9176ee2b005eb8db52027271c6 in 1037ms, sequenceid=391, compaction requested=false 2024-11-24T08:14:34,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:34,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:34,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-24T08:14:34,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-24T08:14:34,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-24T08:14:34,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6530 sec 2024-11-24T08:14:34,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.6570 sec 2024-11-24T08:14:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:34,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:14:34,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:34,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:34,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:34,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:34,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:34,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:34,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112425b1d33e0eef4a79bdba63767fc6701c_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436074685/Put/seqid=0 2024-11-24T08:14:34,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742040_1216 (size=14994) 2024-11-24T08:14:34,706 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:34,710 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112425b1d33e0eef4a79bdba63767fc6701c_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112425b1d33e0eef4a79bdba63767fc6701c_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:34,711 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/99ae08fcc5de4c24bd6b21bd4c7c962f, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:34,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/99ae08fcc5de4c24bd6b21bd4c7c962f is 175, key is test_row_0/A:col10/1732436074685/Put/seqid=0 2024-11-24T08:14:34,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742041_1217 (size=39949) 2024-11-24T08:14:34,724 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=403, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/99ae08fcc5de4c24bd6b21bd4c7c962f 2024-11-24T08:14:34,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732436074712, max timestamp=1732436074728Num uniques 859; 2024-11-24T08:14:34,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732436074712, max timestamp=1732436074728Num uniques 859; 2024-11-24T08:14:34,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732436074712, max timestamp=1732436074728Num uniques 859; 2024-11-24T08:14:34,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T08:14:34,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T08:14:34,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T08:14:34,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/f9ed50983dc74a99a979944eba4e7d94 is 50, key is test_row_0/B:col10/1732436074685/Put/seqid=0 2024-11-24T08:14:34,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742042_1218 (size=12301) 2024-11-24T08:14:34,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436134760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:34,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:34,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436134764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:34,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436134764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:34,798 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T08:14:34,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436134870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:34,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:34,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436134880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:34,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436134880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:34,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-24T08:14:34,971 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:34,971 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-24T08:14:34,973 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:34,974 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:34,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T08:14:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T08:14:35,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436135080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:35,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:35,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436135089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:35,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436135090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:35,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-24T08:14:35,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:35,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:35,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:35,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:35,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:35,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:35,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/f9ed50983dc74a99a979944eba4e7d94 2024-11-24T08:14:35,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/721c8ca7d899447aa4cdfe72853570ea is 50, key is test_row_0/C:col10/1732436074685/Put/seqid=0 2024-11-24T08:14:35,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742043_1219 (size=12301) 2024-11-24T08:14:35,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-24T08:14:35,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:35,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:35,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:35,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:35,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:35,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T08:14:35,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436135390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:35,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436135399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:35,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436135410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:35,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-24T08:14:35,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:35,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:35,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:35,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:35,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:35,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/721c8ca7d899447aa4cdfe72853570ea 2024-11-24T08:14:35,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-24T08:14:35,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:35,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:35,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:35,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:35,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:35,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:35,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/99ae08fcc5de4c24bd6b21bd4c7c962f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/99ae08fcc5de4c24bd6b21bd4c7c962f 2024-11-24T08:14:35,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/99ae08fcc5de4c24bd6b21bd4c7c962f, entries=200, sequenceid=403, filesize=39.0 K 2024-11-24T08:14:35,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/f9ed50983dc74a99a979944eba4e7d94 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/f9ed50983dc74a99a979944eba4e7d94 2024-11-24T08:14:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T08:14:35,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/f9ed50983dc74a99a979944eba4e7d94, entries=150, sequenceid=403, filesize=12.0 K 2024-11-24T08:14:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/721c8ca7d899447aa4cdfe72853570ea as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/721c8ca7d899447aa4cdfe72853570ea 2024-11-24T08:14:35,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/721c8ca7d899447aa4cdfe72853570ea, entries=150, sequenceid=403, filesize=12.0 K 2024-11-24T08:14:35,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 0e4dbe9176ee2b005eb8db52027271c6 in 929ms, sequenceid=403, compaction requested=true 2024-11-24T08:14:35,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:35,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:35,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:35,616 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:35,616 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:35,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:35,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:35,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:35,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:35,618 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:35,618 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:35,618 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/A is initiating minor compaction (all files) 2024-11-24T08:14:35,618 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/B is initiating minor compaction (all files) 2024-11-24T08:14:35,618 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/B in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:35,618 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/A in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:35,618 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/e7a08f8a15f84fb6bb42b3d0ea8c1310, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/cb42c2b09aaa4c4fa19f51217c4e9f18, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/f9ed50983dc74a99a979944eba4e7d94] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=36.7 K 2024-11-24T08:14:35,618 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/7c45cb6cc6ac415d94f63aa1acbccc23, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/424b85d251ff4484a4171f5bb520925c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/99ae08fcc5de4c24bd6b21bd4c7c962f] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=100.8 K 2024-11-24T08:14:35,618 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:35,618 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/7c45cb6cc6ac415d94f63aa1acbccc23, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/424b85d251ff4484a4171f5bb520925c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/99ae08fcc5de4c24bd6b21bd4c7c962f] 2024-11-24T08:14:35,618 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting e7a08f8a15f84fb6bb42b3d0ea8c1310, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732436072245 2024-11-24T08:14:35,619 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c45cb6cc6ac415d94f63aa1acbccc23, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732436072245 2024-11-24T08:14:35,619 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting cb42c2b09aaa4c4fa19f51217c4e9f18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732436072351 2024-11-24T08:14:35,619 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 424b85d251ff4484a4171f5bb520925c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732436072351 2024-11-24T08:14:35,619 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting f9ed50983dc74a99a979944eba4e7d94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732436073518 2024-11-24T08:14:35,620 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99ae08fcc5de4c24bd6b21bd4c7c962f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732436073517 2024-11-24T08:14:35,628 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#B#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:35,628 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/9e403eafbc684b80bfecb23f44211c6c is 50, key is test_row_0/B:col10/1732436074685/Put/seqid=0 2024-11-24T08:14:35,630 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:35,642 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124a48521a26e7f4b658c6b87bf89b8111a_0e4dbe9176ee2b005eb8db52027271c6 store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:35,645 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124a48521a26e7f4b658c6b87bf89b8111a_0e4dbe9176ee2b005eb8db52027271c6, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:35,645 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a48521a26e7f4b658c6b87bf89b8111a_0e4dbe9176ee2b005eb8db52027271c6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:35,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742044_1220 (size=13119) 2024-11-24T08:14:35,652 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/9e403eafbc684b80bfecb23f44211c6c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/9e403eafbc684b80bfecb23f44211c6c 2024-11-24T08:14:35,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742045_1221 (size=4469) 2024-11-24T08:14:35,657 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/B of 0e4dbe9176ee2b005eb8db52027271c6 into 9e403eafbc684b80bfecb23f44211c6c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:35,657 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:35,657 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/B, priority=13, startTime=1732436075616; duration=0sec 2024-11-24T08:14:35,657 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:35,657 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:B 2024-11-24T08:14:35,657 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:35,658 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:35,658 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/C is initiating minor compaction (all files) 2024-11-24T08:14:35,658 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/C in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:35,659 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/8282676917b743a98f42332c9735ea78, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/ae9a1faa1bd7472cae6108aaf8c7995a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/721c8ca7d899447aa4cdfe72853570ea] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=36.7 K 2024-11-24T08:14:35,659 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 8282676917b743a98f42332c9735ea78, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1732436072245 2024-11-24T08:14:35,659 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting ae9a1faa1bd7472cae6108aaf8c7995a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732436072351 2024-11-24T08:14:35,660 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 721c8ca7d899447aa4cdfe72853570ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732436073518 2024-11-24T08:14:35,666 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#C#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:35,666 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/c7a2a791712340c086df0baff68ec512 is 50, key is test_row_0/C:col10/1732436074685/Put/seqid=0 2024-11-24T08:14:35,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742046_1222 (size=13119) 2024-11-24T08:14:35,677 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/c7a2a791712340c086df0baff68ec512 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/c7a2a791712340c086df0baff68ec512 2024-11-24T08:14:35,685 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/C of 0e4dbe9176ee2b005eb8db52027271c6 into c7a2a791712340c086df0baff68ec512(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:35,685 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:35,685 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/C, priority=13, startTime=1732436075616; duration=0sec 2024-11-24T08:14:35,685 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:35,685 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:C 2024-11-24T08:14:35,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-24T08:14:35,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:35,743 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-11-24T08:14:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:35,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:35,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112437fa5dca58b34c67972618b5c91ef0c1_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436074762/Put/seqid=0 2024-11-24T08:14:35,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742047_1223 (size=12454) 2024-11-24T08:14:35,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:35,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:35,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:35,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436135933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:35,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:35,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436135936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:35,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:35,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436135937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:35,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:35,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:36,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:36,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436136050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:36,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436136050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:36,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:36,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:36,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436136050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:36,057 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#A#compaction#181 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:36,060 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/6c5ee3956dcc406fb5fa3f7fde13aad3 is 175, key is test_row_0/A:col10/1732436074685/Put/seqid=0 2024-11-24T08:14:36,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742048_1224 (size=32073) 2024-11-24T08:14:36,093 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/6c5ee3956dcc406fb5fa3f7fde13aad3 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/6c5ee3956dcc406fb5fa3f7fde13aad3 2024-11-24T08:14:36,104 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/A of 0e4dbe9176ee2b005eb8db52027271c6 into 6c5ee3956dcc406fb5fa3f7fde13aad3(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:36,104 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:36,104 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/A, priority=13, startTime=1732436075615; duration=0sec 2024-11-24T08:14:36,105 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:36,105 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:A 2024-11-24T08:14:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T08:14:36,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:36,185 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112437fa5dca58b34c67972618b5c91ef0c1_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112437fa5dca58b34c67972618b5c91ef0c1_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:36,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/3ee1eb34ded24dfb8186d58b29a4ddf9, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/3ee1eb34ded24dfb8186d58b29a4ddf9 is 175, key is test_row_0/A:col10/1732436074762/Put/seqid=0 2024-11-24T08:14:36,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742049_1225 (size=31255) 2024-11-24T08:14:36,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436136260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:36,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:36,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436136260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:36,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:36,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436136260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:36,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:36,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436136570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:36,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436136570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436136570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:36,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:36,599 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=439, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/3ee1eb34ded24dfb8186d58b29a4ddf9 2024-11-24T08:14:36,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/c3db3b1dc5cc48828c7006d853182405 is 50, key is test_row_0/B:col10/1732436074762/Put/seqid=0 2024-11-24T08:14:36,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742050_1226 (size=12301) 2024-11-24T08:14:36,622 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/c3db3b1dc5cc48828c7006d853182405 2024-11-24T08:14:36,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/70169d78cc9e47e9987941b9a1347aff is 50, key is test_row_0/C:col10/1732436074762/Put/seqid=0 2024-11-24T08:14:36,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742051_1227 (size=12301) 2024-11-24T08:14:36,658 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/70169d78cc9e47e9987941b9a1347aff 2024-11-24T08:14:36,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/3ee1eb34ded24dfb8186d58b29a4ddf9 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/3ee1eb34ded24dfb8186d58b29a4ddf9 2024-11-24T08:14:36,683 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/3ee1eb34ded24dfb8186d58b29a4ddf9, entries=150, sequenceid=439, filesize=30.5 K 2024-11-24T08:14:36,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/c3db3b1dc5cc48828c7006d853182405 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/c3db3b1dc5cc48828c7006d853182405 2024-11-24T08:14:36,697 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/c3db3b1dc5cc48828c7006d853182405, entries=150, sequenceid=439, filesize=12.0 K 2024-11-24T08:14:36,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/70169d78cc9e47e9987941b9a1347aff as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/70169d78cc9e47e9987941b9a1347aff 2024-11-24T08:14:36,704 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/70169d78cc9e47e9987941b9a1347aff, entries=150, sequenceid=439, filesize=12.0 K 2024-11-24T08:14:36,707 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 0e4dbe9176ee2b005eb8db52027271c6 in 963ms, sequenceid=439, compaction requested=false 2024-11-24T08:14:36,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:36,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:36,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-24T08:14:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-24T08:14:36,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-24T08:14:36,714 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7350 sec 2024-11-24T08:14:36,716 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.7430 sec 2024-11-24T08:14:37,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:37,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T08:14:37,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:37,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:37,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:37,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:37,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:37,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:37,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124cc3314163ea940669fd7551276613e9b_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436077074/Put/seqid=0 2024-11-24T08:14:37,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742052_1228 (size=12454) 2024-11-24T08:14:37,096 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:37,101 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124cc3314163ea940669fd7551276613e9b_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cc3314163ea940669fd7551276613e9b_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:37,103 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/c5e1db96020743fa9149928676718c62, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:37,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/c5e1db96020743fa9149928676718c62 is 175, key is test_row_0/A:col10/1732436077074/Put/seqid=0 2024-11-24T08:14:37,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436137107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436137108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436137109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742053_1229 (size=31255) 2024-11-24T08:14:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436137115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-24T08:14:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,120 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:37,122 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:37,123 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=452, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/c5e1db96020743fa9149928676718c62 2024-11-24T08:14:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-24T08:14:37,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436137117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T08:14:37,124 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:37,125 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:37,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:37,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/410b170fd45e4b9eb9b6ecc4688c1348 is 50, key is test_row_0/B:col10/1732436077074/Put/seqid=0 2024-11-24T08:14:37,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742054_1230 (size=12301) 2024-11-24T08:14:37,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436137220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T08:14:37,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436137230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436137230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436137231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436137231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-24T08:14:37,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:37,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:37,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:37,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:37,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:37,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60130 deadline: 1732436137429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-24T08:14:37,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:37,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:37,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:37,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T08:14:37,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60194 deadline: 1732436137450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60164 deadline: 1732436137450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60170 deadline: 1732436137450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:37,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60134 deadline: 1732436137450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:37,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:37,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:14:37,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/410b170fd45e4b9eb9b6ecc4688c1348 2024-11-24T08:14:37,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/b584087a5cca42eda6d10f40e6d8a68b is 50, key is test_row_0/C:col10/1732436077074/Put/seqid=0 2024-11-24T08:14:37,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742055_1231 (size=12301) 2024-11-24T08:14:37,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/b584087a5cca42eda6d10f40e6d8a68b 2024-11-24T08:14:37,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-24T08:14:37,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:37,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:37,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:37,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:37,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:37,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:37,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/c5e1db96020743fa9149928676718c62 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/c5e1db96020743fa9149928676718c62 2024-11-24T08:14:37,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/c5e1db96020743fa9149928676718c62, entries=150, sequenceid=452, filesize=30.5 K 2024-11-24T08:14:37,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/410b170fd45e4b9eb9b6ecc4688c1348 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/410b170fd45e4b9eb9b6ecc4688c1348 2024-11-24T08:14:37,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/410b170fd45e4b9eb9b6ecc4688c1348, entries=150, sequenceid=452, filesize=12.0 K 2024-11-24T08:14:37,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/b584087a5cca42eda6d10f40e6d8a68b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/b584087a5cca42eda6d10f40e6d8a68b 2024-11-24T08:14:37,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/b584087a5cca42eda6d10f40e6d8a68b, entries=150, sequenceid=452, filesize=12.0 K 2024-11-24T08:14:37,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 0e4dbe9176ee2b005eb8db52027271c6 in 542ms, sequenceid=452, compaction requested=true 2024-11-24T08:14:37,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:37,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:37,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:37,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:37,618 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:37,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:37,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0e4dbe9176ee2b005eb8db52027271c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:37,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:37,618 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:37,619 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:37,619 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/A is initiating minor compaction (all files) 2024-11-24T08:14:37,619 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:37,619 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/A in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:37,619 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/B is initiating minor compaction (all files) 2024-11-24T08:14:37,619 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/6c5ee3956dcc406fb5fa3f7fde13aad3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/3ee1eb34ded24dfb8186d58b29a4ddf9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/c5e1db96020743fa9149928676718c62] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=92.4 K 2024-11-24T08:14:37,619 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/B in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:37,619 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:37,619 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/9e403eafbc684b80bfecb23f44211c6c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/c3db3b1dc5cc48828c7006d853182405, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/410b170fd45e4b9eb9b6ecc4688c1348] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=36.8 K 2024-11-24T08:14:37,619 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/6c5ee3956dcc406fb5fa3f7fde13aad3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/3ee1eb34ded24dfb8186d58b29a4ddf9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/c5e1db96020743fa9149928676718c62] 2024-11-24T08:14:37,620 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e403eafbc684b80bfecb23f44211c6c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732436073518 2024-11-24T08:14:37,620 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c5ee3956dcc406fb5fa3f7fde13aad3, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732436073518 2024-11-24T08:14:37,621 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting c3db3b1dc5cc48828c7006d853182405, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732436074755 2024-11-24T08:14:37,621 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ee1eb34ded24dfb8186d58b29a4ddf9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732436074755 2024-11-24T08:14:37,621 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 410b170fd45e4b9eb9b6ecc4688c1348, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732436075917 2024-11-24T08:14:37,622 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5e1db96020743fa9149928676718c62, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732436075917 2024-11-24T08:14:37,629 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:37,629 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#B#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:37,630 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/41297cd0560746a7bdfff2950b90d6b5 is 50, key is test_row_0/B:col10/1732436077074/Put/seqid=0 2024-11-24T08:14:37,641 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411243f5be3f9999d485698ceb4c0c096ff3c_0e4dbe9176ee2b005eb8db52027271c6 store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:37,643 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411243f5be3f9999d485698ceb4c0c096ff3c_0e4dbe9176ee2b005eb8db52027271c6, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:37,643 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243f5be3f9999d485698ceb4c0c096ff3c_0e4dbe9176ee2b005eb8db52027271c6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:37,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742056_1232 (size=13221) 2024-11-24T08:14:37,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742057_1233 (size=4469) 2024-11-24T08:14:37,734 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-24T08:14:37,735 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:37,735 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,735 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,735 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:37,737 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-11-24T08:14:37,738 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:37,738 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,738 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:37,738 INFO [Thread-640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-640. 2024-11-24T08:14:37,738 DEBUG [Thread-640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:37,738 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,738 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,739 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:37,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-24T08:14:37,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:37,739 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T08:14:37,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:37,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:37,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:37,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:37,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:37,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:37,741 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-24T08:14:37,741 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:37,741 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,741 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,741 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:37,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. as already flushing 2024-11-24T08:14:37,742 INFO [Thread-627 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-627. 2024-11-24T08:14:37,742 DEBUG [Thread-627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:37,742 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,742 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,742 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:37,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240ac85604b857468c945880f214b0d207_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436077108/Put/seqid=0 2024-11-24T08:14:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T08:14:37,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742058_1234 (size=12454) 2024-11-24T08:14:37,762 INFO [Thread-623 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-623. 2024-11-24T08:14:37,762 INFO [Thread-625 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-625. 2024-11-24T08:14:37,762 INFO [Thread-629 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-629. 2024-11-24T08:14:37,762 DEBUG [Thread-629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:37,762 DEBUG [Thread-623 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:37,762 DEBUG [Thread-625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:37,762 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,762 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,762 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,762 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,762 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,762 DEBUG [Thread-623 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:37,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:37,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:37,763 INFO [Thread-631 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-631. 2024-11-24T08:14:37,763 DEBUG [Thread-631 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:14:37,763 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,763 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:37,763 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:38,052 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/41297cd0560746a7bdfff2950b90d6b5 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/41297cd0560746a7bdfff2950b90d6b5 2024-11-24T08:14:38,055 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#A#compaction#190 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:38,055 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/9e782337e7aa46469c481331ef056b99 is 175, key is test_row_0/A:col10/1732436077074/Put/seqid=0 2024-11-24T08:14:38,057 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/B of 0e4dbe9176ee2b005eb8db52027271c6 into 41297cd0560746a7bdfff2950b90d6b5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:38,057 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:38,057 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/B, priority=13, startTime=1732436077618; duration=0sec 2024-11-24T08:14:38,057 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:38,057 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:B 2024-11-24T08:14:38,057 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:38,058 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:38,058 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 0e4dbe9176ee2b005eb8db52027271c6/C is initiating minor compaction (all files) 2024-11-24T08:14:38,058 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0e4dbe9176ee2b005eb8db52027271c6/C in TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:38,058 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/c7a2a791712340c086df0baff68ec512, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/70169d78cc9e47e9987941b9a1347aff, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/b584087a5cca42eda6d10f40e6d8a68b] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp, totalSize=36.8 K 2024-11-24T08:14:38,059 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting c7a2a791712340c086df0baff68ec512, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732436073518 2024-11-24T08:14:38,059 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 70169d78cc9e47e9987941b9a1347aff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1732436074755 2024-11-24T08:14:38,060 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting b584087a5cca42eda6d10f40e6d8a68b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732436075917 2024-11-24T08:14:38,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742059_1235 (size=32175) 2024-11-24T08:14:38,067 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/9e782337e7aa46469c481331ef056b99 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/9e782337e7aa46469c481331ef056b99 2024-11-24T08:14:38,067 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0e4dbe9176ee2b005eb8db52027271c6#C#compaction#192 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:38,068 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/c0710971780d4fe7918d6485bdc3b42a is 50, key is test_row_0/C:col10/1732436077074/Put/seqid=0 2024-11-24T08:14:38,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742060_1236 (size=13221) 2024-11-24T08:14:38,072 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/A of 0e4dbe9176ee2b005eb8db52027271c6 into 9e782337e7aa46469c481331ef056b99(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:38,072 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:38,072 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/A, priority=13, startTime=1732436077617; duration=0sec 2024-11-24T08:14:38,072 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:38,073 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:A 2024-11-24T08:14:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:38,157 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240ac85604b857468c945880f214b0d207_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240ac85604b857468c945880f214b0d207_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:38,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/43ccf60eec2447c49e47fed7e1f5790f, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:38,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/43ccf60eec2447c49e47fed7e1f5790f is 175, key is test_row_0/A:col10/1732436077108/Put/seqid=0 2024-11-24T08:14:38,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742061_1237 (size=31255) 2024-11-24T08:14:38,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T08:14:38,477 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/c0710971780d4fe7918d6485bdc3b42a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/c0710971780d4fe7918d6485bdc3b42a 2024-11-24T08:14:38,481 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0e4dbe9176ee2b005eb8db52027271c6/C of 0e4dbe9176ee2b005eb8db52027271c6 into c0710971780d4fe7918d6485bdc3b42a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:38,481 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:38,481 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., storeName=0e4dbe9176ee2b005eb8db52027271c6/C, priority=13, startTime=1732436077618; duration=0sec 2024-11-24T08:14:38,481 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:38,481 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0e4dbe9176ee2b005eb8db52027271c6:C 2024-11-24T08:14:38,563 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=472, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/43ccf60eec2447c49e47fed7e1f5790f 2024-11-24T08:14:38,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/4e589f7b49b04484bfe6b47108920930 is 50, key is test_row_0/B:col10/1732436077108/Put/seqid=0 2024-11-24T08:14:38,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742062_1238 (size=12301) 2024-11-24T08:14:38,983 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/4e589f7b49b04484bfe6b47108920930 2024-11-24T08:14:38,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/ab324b55aa984efa93233afe3f6ff1be is 50, key is test_row_0/C:col10/1732436077108/Put/seqid=0 2024-11-24T08:14:38,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742063_1239 (size=12301) 2024-11-24T08:14:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T08:14:39,397 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/ab324b55aa984efa93233afe3f6ff1be 2024-11-24T08:14:39,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/43ccf60eec2447c49e47fed7e1f5790f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/43ccf60eec2447c49e47fed7e1f5790f 2024-11-24T08:14:39,408 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/43ccf60eec2447c49e47fed7e1f5790f, entries=150, sequenceid=472, filesize=30.5 K 2024-11-24T08:14:39,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/4e589f7b49b04484bfe6b47108920930 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/4e589f7b49b04484bfe6b47108920930 2024-11-24T08:14:39,415 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/4e589f7b49b04484bfe6b47108920930, entries=150, sequenceid=472, filesize=12.0 K 2024-11-24T08:14:39,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/ab324b55aa984efa93233afe3f6ff1be as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/ab324b55aa984efa93233afe3f6ff1be 2024-11-24T08:14:39,422 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/ab324b55aa984efa93233afe3f6ff1be, entries=150, sequenceid=472, filesize=12.0 K 2024-11-24T08:14:39,423 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=33.54 KB/34350 for 0e4dbe9176ee2b005eb8db52027271c6 in 1684ms, sequenceid=472, compaction requested=false 2024-11-24T08:14:39,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 0e4dbe9176ee2b005eb8db52027271c6: 2024-11-24T08:14:39,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:39,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-24T08:14:39,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-24T08:14:39,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-24T08:14:39,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3000 sec 2024-11-24T08:14:39,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.3060 sec 2024-11-24T08:14:41,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-24T08:14:41,291 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:41,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T08:14:41,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 137 2024-11-24T08:14:41,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-24T08:14:41,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-24T08:14:41,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-24T08:14:41,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-24T08:14:41,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T08:14:41,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2507 2024-11-24T08:14:41,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2383 2024-11-24T08:14:41,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T08:14:41,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1078 2024-11-24T08:14:41,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3233 rows 2024-11-24T08:14:41,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1072 2024-11-24T08:14:41,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3212 rows 2024-11-24T08:14:41,291 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T08:14:41,291 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:14:41,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:41,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:41,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:14:41,292 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T08:14:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:41,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T08:14:41,296 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436081295"}]},"ts":"1732436081295"} 2024-11-24T08:14:41,297 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T08:14:41,297 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T08:14:41,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T08:14:41,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, UNASSIGN}] 2024-11-24T08:14:41,300 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, UNASSIGN 2024-11-24T08:14:41,301 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=0e4dbe9176ee2b005eb8db52027271c6, regionState=CLOSING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:14:41,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, UNASSIGN because future has completed 2024-11-24T08:14:41,303 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T08:14:41,303 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:14:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T08:14:41,455 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:41,455 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T08:14:41,455 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 0e4dbe9176ee2b005eb8db52027271c6, disabling compactions & flushes 2024-11-24T08:14:41,455 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:41,456 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:41,456 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. after waiting 0 ms 2024-11-24T08:14:41,456 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:41,456 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 0e4dbe9176ee2b005eb8db52027271c6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T08:14:41,456 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=A 2024-11-24T08:14:41,456 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:41,456 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=B 2024-11-24T08:14:41,456 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:41,456 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 0e4dbe9176ee2b005eb8db52027271c6, store=C 2024-11-24T08:14:41,456 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:41,464 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112412b024be616b4d88927d72e9fe9acae3_0e4dbe9176ee2b005eb8db52027271c6 is 50, key is test_row_0/A:col10/1732436077760/Put/seqid=0 2024-11-24T08:14:41,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742064_1240 (size=9914) 2024-11-24T08:14:41,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T08:14:41,870 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:41,875 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112412b024be616b4d88927d72e9fe9acae3_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112412b024be616b4d88927d72e9fe9acae3_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:41,876 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/c90661d1f8674904b1a20f0e9289eb3f, store: [table=TestAcidGuarantees family=A region=0e4dbe9176ee2b005eb8db52027271c6] 2024-11-24T08:14:41,877 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/c90661d1f8674904b1a20f0e9289eb3f is 175, key is test_row_0/A:col10/1732436077760/Put/seqid=0 2024-11-24T08:14:41,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742065_1241 (size=22561) 2024-11-24T08:14:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T08:14:42,284 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=483, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/c90661d1f8674904b1a20f0e9289eb3f 2024-11-24T08:14:42,293 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/a92b441155e445859d0234b962409761 is 50, key is test_row_0/B:col10/1732436077760/Put/seqid=0 2024-11-24T08:14:42,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742066_1242 (size=9857) 2024-11-24T08:14:42,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T08:14:42,699 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/a92b441155e445859d0234b962409761 2024-11-24T08:14:42,707 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/eb4b6987e63a419b9dc37a449ac775a9 is 50, key is test_row_0/C:col10/1732436077760/Put/seqid=0 2024-11-24T08:14:42,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742067_1243 (size=9857) 2024-11-24T08:14:43,113 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/eb4b6987e63a419b9dc37a449ac775a9 2024-11-24T08:14:43,118 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/A/c90661d1f8674904b1a20f0e9289eb3f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/c90661d1f8674904b1a20f0e9289eb3f 2024-11-24T08:14:43,122 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/c90661d1f8674904b1a20f0e9289eb3f, entries=100, sequenceid=483, filesize=22.0 K 2024-11-24T08:14:43,123 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/B/a92b441155e445859d0234b962409761 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/a92b441155e445859d0234b962409761 2024-11-24T08:14:43,126 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/a92b441155e445859d0234b962409761, entries=100, sequenceid=483, filesize=9.6 K 2024-11-24T08:14:43,127 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/.tmp/C/eb4b6987e63a419b9dc37a449ac775a9 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/eb4b6987e63a419b9dc37a449ac775a9 2024-11-24T08:14:43,131 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/eb4b6987e63a419b9dc37a449ac775a9, entries=100, sequenceid=483, filesize=9.6 K 2024-11-24T08:14:43,132 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0e4dbe9176ee2b005eb8db52027271c6 in 1676ms, sequenceid=483, compaction requested=true 2024-11-24T08:14:43,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/d981deb55fce4fc4aaf8a7eedc77f6bc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/3e2d6ccb9889484caca2f23da7c2e8a3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/579402c15de24308a3c990af79b72763, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/67ddba911bf947e2896026f575a23fa0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/75b3224c141440ddaf4c52216c70bc89, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/1d0507764916430e9ceabf78c4a32270, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/d80c197c79254489a6b9c080bacfcb0f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/46c9b65f426847ffb18dd8903bba474f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/ccf4eca1de4f4bcca553adc24e78378d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/2f17da791b4e4eb696ec177eeeff338b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/35ea893af27e4b65933065beb710179f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/15a79822e4df4adf8352c38853f567f0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/11b086ce04324f58b08b7421c5bed46f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/323afd8881d4413d86e9220210389605, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/b958897b2ef14534831e0badc8740db2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/caa500ffbb77440691f5871f3bf0099b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/761cf8fd3f9b434399d60d6532748bb0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/e285f5867731477ca91a83af914f4850, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/81de9901a1f240a79aa8fe662165fa81, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/a2e9baa39a2548678a109af274e63b0d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/bb26dcf7e5bd44dea47f6c6cf2337635, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/7c45cb6cc6ac415d94f63aa1acbccc23, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/424b85d251ff4484a4171f5bb520925c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/99ae08fcc5de4c24bd6b21bd4c7c962f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/6c5ee3956dcc406fb5fa3f7fde13aad3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/3ee1eb34ded24dfb8186d58b29a4ddf9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/c5e1db96020743fa9149928676718c62] to archive 2024-11-24T08:14:43,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:14:43,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/d981deb55fce4fc4aaf8a7eedc77f6bc to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/d981deb55fce4fc4aaf8a7eedc77f6bc 2024-11-24T08:14:43,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/3e2d6ccb9889484caca2f23da7c2e8a3 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/3e2d6ccb9889484caca2f23da7c2e8a3 2024-11-24T08:14:43,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/579402c15de24308a3c990af79b72763 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/579402c15de24308a3c990af79b72763 2024-11-24T08:14:43,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/67ddba911bf947e2896026f575a23fa0 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/67ddba911bf947e2896026f575a23fa0 2024-11-24T08:14:43,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/75b3224c141440ddaf4c52216c70bc89 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/75b3224c141440ddaf4c52216c70bc89 2024-11-24T08:14:43,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/1d0507764916430e9ceabf78c4a32270 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/1d0507764916430e9ceabf78c4a32270 2024-11-24T08:14:43,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/d80c197c79254489a6b9c080bacfcb0f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/d80c197c79254489a6b9c080bacfcb0f 2024-11-24T08:14:43,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/46c9b65f426847ffb18dd8903bba474f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/46c9b65f426847ffb18dd8903bba474f 2024-11-24T08:14:43,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/ccf4eca1de4f4bcca553adc24e78378d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/ccf4eca1de4f4bcca553adc24e78378d 2024-11-24T08:14:43,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/2f17da791b4e4eb696ec177eeeff338b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/2f17da791b4e4eb696ec177eeeff338b 2024-11-24T08:14:43,151 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/35ea893af27e4b65933065beb710179f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/35ea893af27e4b65933065beb710179f 2024-11-24T08:14:43,153 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/15a79822e4df4adf8352c38853f567f0 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/15a79822e4df4adf8352c38853f567f0 2024-11-24T08:14:43,154 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/11b086ce04324f58b08b7421c5bed46f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/11b086ce04324f58b08b7421c5bed46f 2024-11-24T08:14:43,155 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/323afd8881d4413d86e9220210389605 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/323afd8881d4413d86e9220210389605 2024-11-24T08:14:43,156 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/b958897b2ef14534831e0badc8740db2 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/b958897b2ef14534831e0badc8740db2 2024-11-24T08:14:43,157 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/caa500ffbb77440691f5871f3bf0099b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/caa500ffbb77440691f5871f3bf0099b 2024-11-24T08:14:43,159 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/761cf8fd3f9b434399d60d6532748bb0 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/761cf8fd3f9b434399d60d6532748bb0 2024-11-24T08:14:43,163 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/e285f5867731477ca91a83af914f4850 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/e285f5867731477ca91a83af914f4850 2024-11-24T08:14:43,165 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/81de9901a1f240a79aa8fe662165fa81 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/81de9901a1f240a79aa8fe662165fa81 2024-11-24T08:14:43,167 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/a2e9baa39a2548678a109af274e63b0d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/a2e9baa39a2548678a109af274e63b0d 2024-11-24T08:14:43,168 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/bb26dcf7e5bd44dea47f6c6cf2337635 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/bb26dcf7e5bd44dea47f6c6cf2337635 2024-11-24T08:14:43,169 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/7c45cb6cc6ac415d94f63aa1acbccc23 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/7c45cb6cc6ac415d94f63aa1acbccc23 2024-11-24T08:14:43,170 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/424b85d251ff4484a4171f5bb520925c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/424b85d251ff4484a4171f5bb520925c 2024-11-24T08:14:43,171 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/99ae08fcc5de4c24bd6b21bd4c7c962f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/99ae08fcc5de4c24bd6b21bd4c7c962f 2024-11-24T08:14:43,172 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/6c5ee3956dcc406fb5fa3f7fde13aad3 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/6c5ee3956dcc406fb5fa3f7fde13aad3 2024-11-24T08:14:43,173 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/3ee1eb34ded24dfb8186d58b29a4ddf9 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/3ee1eb34ded24dfb8186d58b29a4ddf9 2024-11-24T08:14:43,174 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/c5e1db96020743fa9149928676718c62 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/c5e1db96020743fa9149928676718c62 2024-11-24T08:14:43,176 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/2641b472fa514874b3a1ea60f64256e5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/de79b28572f74d7dad9fc4801b16f2fa, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/d9564990f5954f04b6d5e27eba7afea7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7af74a6ff6814ba2a3b12be97bf22c5d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ebff6f54ff6f4d37be2b39284a4d7cdd, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7fef601ef21242efb80d14dc27fabc60, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/25433e6fe0ff4757bb550bd603ac9157, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/95b63fdb3d60485aa85ec33b95011c14, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7f21d489f42a4e178825cf3916f5beec, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7d9c7a47019945c0b813fd602de6d1e4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/adc3e825e3034cda81ac6b260c6b68a1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/899268ab3ab04235b49d1f40333001af, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/38ad01f74e0b42a8ad2ae1e7fd9a3928, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ad678ca5c818406787fc31c15399fcc9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/69f713f3c4f1496c8fc5dbae15563960, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/89b809dc767f4c9e81b7fabd9f2925f7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ebe202bcba464ee89b05aaa8839ecc1e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/aec89cec8ee94211ae87bd6dd19ee9ad, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/c74f7facd6a54fa4871259819b279dad, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/9ef4dd0631d5495c94fd30085d8a9a45, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/e7a08f8a15f84fb6bb42b3d0ea8c1310, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/0dafce8c8d244d62906d665443e30853, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/cb42c2b09aaa4c4fa19f51217c4e9f18, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/9e403eafbc684b80bfecb23f44211c6c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/f9ed50983dc74a99a979944eba4e7d94, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/c3db3b1dc5cc48828c7006d853182405, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/410b170fd45e4b9eb9b6ecc4688c1348] to archive 2024-11-24T08:14:43,177 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:14:43,178 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/2641b472fa514874b3a1ea60f64256e5 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/2641b472fa514874b3a1ea60f64256e5 2024-11-24T08:14:43,179 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/de79b28572f74d7dad9fc4801b16f2fa to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/de79b28572f74d7dad9fc4801b16f2fa 2024-11-24T08:14:43,181 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/d9564990f5954f04b6d5e27eba7afea7 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/d9564990f5954f04b6d5e27eba7afea7 2024-11-24T08:14:43,182 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7af74a6ff6814ba2a3b12be97bf22c5d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7af74a6ff6814ba2a3b12be97bf22c5d 2024-11-24T08:14:43,183 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ebff6f54ff6f4d37be2b39284a4d7cdd to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ebff6f54ff6f4d37be2b39284a4d7cdd 2024-11-24T08:14:43,184 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7fef601ef21242efb80d14dc27fabc60 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7fef601ef21242efb80d14dc27fabc60 2024-11-24T08:14:43,185 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/25433e6fe0ff4757bb550bd603ac9157 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/25433e6fe0ff4757bb550bd603ac9157 2024-11-24T08:14:43,186 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/95b63fdb3d60485aa85ec33b95011c14 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/95b63fdb3d60485aa85ec33b95011c14 2024-11-24T08:14:43,188 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7f21d489f42a4e178825cf3916f5beec to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7f21d489f42a4e178825cf3916f5beec 2024-11-24T08:14:43,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7d9c7a47019945c0b813fd602de6d1e4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/7d9c7a47019945c0b813fd602de6d1e4 2024-11-24T08:14:43,191 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/adc3e825e3034cda81ac6b260c6b68a1 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/adc3e825e3034cda81ac6b260c6b68a1 2024-11-24T08:14:43,192 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/899268ab3ab04235b49d1f40333001af to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/899268ab3ab04235b49d1f40333001af 2024-11-24T08:14:43,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/38ad01f74e0b42a8ad2ae1e7fd9a3928 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/38ad01f74e0b42a8ad2ae1e7fd9a3928 2024-11-24T08:14:43,195 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ad678ca5c818406787fc31c15399fcc9 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ad678ca5c818406787fc31c15399fcc9 2024-11-24T08:14:43,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/69f713f3c4f1496c8fc5dbae15563960 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/69f713f3c4f1496c8fc5dbae15563960 2024-11-24T08:14:43,197 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/89b809dc767f4c9e81b7fabd9f2925f7 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/89b809dc767f4c9e81b7fabd9f2925f7 2024-11-24T08:14:43,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ebe202bcba464ee89b05aaa8839ecc1e to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/ebe202bcba464ee89b05aaa8839ecc1e 2024-11-24T08:14:43,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/aec89cec8ee94211ae87bd6dd19ee9ad to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/aec89cec8ee94211ae87bd6dd19ee9ad 2024-11-24T08:14:43,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/c74f7facd6a54fa4871259819b279dad to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/c74f7facd6a54fa4871259819b279dad 2024-11-24T08:14:43,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/9ef4dd0631d5495c94fd30085d8a9a45 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/9ef4dd0631d5495c94fd30085d8a9a45 2024-11-24T08:14:43,203 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/e7a08f8a15f84fb6bb42b3d0ea8c1310 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/e7a08f8a15f84fb6bb42b3d0ea8c1310 2024-11-24T08:14:43,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/0dafce8c8d244d62906d665443e30853 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/0dafce8c8d244d62906d665443e30853 2024-11-24T08:14:43,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/cb42c2b09aaa4c4fa19f51217c4e9f18 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/cb42c2b09aaa4c4fa19f51217c4e9f18 2024-11-24T08:14:43,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/9e403eafbc684b80bfecb23f44211c6c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/9e403eafbc684b80bfecb23f44211c6c 2024-11-24T08:14:43,208 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/f9ed50983dc74a99a979944eba4e7d94 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/f9ed50983dc74a99a979944eba4e7d94 2024-11-24T08:14:43,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/c3db3b1dc5cc48828c7006d853182405 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/c3db3b1dc5cc48828c7006d853182405 2024-11-24T08:14:43,211 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/410b170fd45e4b9eb9b6ecc4688c1348 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/410b170fd45e4b9eb9b6ecc4688c1348 2024-11-24T08:14:43,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/e553c2f89cfb43c68f0022e6ab63e699, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/46d458bdf09043cb899a172625aaa1a5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/450a0cf343cf4caea24aa0cb9486f0c4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/c5b49f12942842328361301c6c070100, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/e2be7efe32174d0b9c54942ed87c180d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/a3726ce5ce744a8d93012887ebbc334a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/9e2498122e42475bad29dd47bda094b1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/23b3fb706a18422b8508836f4d3e4c96, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/3f5bc05d64f047c08911ac0dce4e79fe, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/5276fe22353547c59e9e2a4ded431846, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/51ca97b53d754dc9bcb928f83a4e84de, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/8e15fb1f717748c4b1fbc643e6d8eaac, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/51d984892b0b46a7a81dbd8d8d9b6ca7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/0be0e95fd37c4f3588266637461826ed, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/1d96912b895e498e8f5a3d88f06e9d20, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/238d4aa162644e24b94fb763282cff7d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/362f62cc63794aa69fdd20875c6894db, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/19b309490d6a4efe9432b3d226cff92e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/612912e606614bf592562ddcc8997201, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/f7147b27a96842dc88b27aabfc6685a9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/8282676917b743a98f42332c9735ea78, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/3879ed8611744658919af71af67c2a22, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/ae9a1faa1bd7472cae6108aaf8c7995a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/c7a2a791712340c086df0baff68ec512, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/721c8ca7d899447aa4cdfe72853570ea, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/70169d78cc9e47e9987941b9a1347aff, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/b584087a5cca42eda6d10f40e6d8a68b] to archive 2024-11-24T08:14:43,218 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:14:43,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/e553c2f89cfb43c68f0022e6ab63e699 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/e553c2f89cfb43c68f0022e6ab63e699 2024-11-24T08:14:43,222 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/46d458bdf09043cb899a172625aaa1a5 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/46d458bdf09043cb899a172625aaa1a5 2024-11-24T08:14:43,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/450a0cf343cf4caea24aa0cb9486f0c4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/450a0cf343cf4caea24aa0cb9486f0c4 2024-11-24T08:14:43,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/c5b49f12942842328361301c6c070100 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/c5b49f12942842328361301c6c070100 2024-11-24T08:14:43,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/e2be7efe32174d0b9c54942ed87c180d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/e2be7efe32174d0b9c54942ed87c180d 2024-11-24T08:14:43,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/a3726ce5ce744a8d93012887ebbc334a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/a3726ce5ce744a8d93012887ebbc334a 2024-11-24T08:14:43,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/9e2498122e42475bad29dd47bda094b1 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/9e2498122e42475bad29dd47bda094b1 2024-11-24T08:14:43,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/23b3fb706a18422b8508836f4d3e4c96 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/23b3fb706a18422b8508836f4d3e4c96 2024-11-24T08:14:43,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/3f5bc05d64f047c08911ac0dce4e79fe to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/3f5bc05d64f047c08911ac0dce4e79fe 2024-11-24T08:14:43,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/5276fe22353547c59e9e2a4ded431846 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/5276fe22353547c59e9e2a4ded431846 2024-11-24T08:14:43,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/51ca97b53d754dc9bcb928f83a4e84de to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/51ca97b53d754dc9bcb928f83a4e84de 2024-11-24T08:14:43,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/8e15fb1f717748c4b1fbc643e6d8eaac to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/8e15fb1f717748c4b1fbc643e6d8eaac 2024-11-24T08:14:43,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/51d984892b0b46a7a81dbd8d8d9b6ca7 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/51d984892b0b46a7a81dbd8d8d9b6ca7 2024-11-24T08:14:43,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/0be0e95fd37c4f3588266637461826ed to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/0be0e95fd37c4f3588266637461826ed 2024-11-24T08:14:43,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/1d96912b895e498e8f5a3d88f06e9d20 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/1d96912b895e498e8f5a3d88f06e9d20 2024-11-24T08:14:43,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/238d4aa162644e24b94fb763282cff7d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/238d4aa162644e24b94fb763282cff7d 2024-11-24T08:14:43,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/362f62cc63794aa69fdd20875c6894db to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/362f62cc63794aa69fdd20875c6894db 2024-11-24T08:14:43,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/19b309490d6a4efe9432b3d226cff92e to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/19b309490d6a4efe9432b3d226cff92e 2024-11-24T08:14:43,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/612912e606614bf592562ddcc8997201 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/612912e606614bf592562ddcc8997201 2024-11-24T08:14:43,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/f7147b27a96842dc88b27aabfc6685a9 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/f7147b27a96842dc88b27aabfc6685a9 2024-11-24T08:14:43,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/8282676917b743a98f42332c9735ea78 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/8282676917b743a98f42332c9735ea78 2024-11-24T08:14:43,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/3879ed8611744658919af71af67c2a22 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/3879ed8611744658919af71af67c2a22 2024-11-24T08:14:43,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/ae9a1faa1bd7472cae6108aaf8c7995a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/ae9a1faa1bd7472cae6108aaf8c7995a 2024-11-24T08:14:43,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/c7a2a791712340c086df0baff68ec512 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/c7a2a791712340c086df0baff68ec512 2024-11-24T08:14:43,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/721c8ca7d899447aa4cdfe72853570ea to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/721c8ca7d899447aa4cdfe72853570ea 2024-11-24T08:14:43,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/70169d78cc9e47e9987941b9a1347aff to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/70169d78cc9e47e9987941b9a1347aff 2024-11-24T08:14:43,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/b584087a5cca42eda6d10f40e6d8a68b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/b584087a5cca42eda6d10f40e6d8a68b 2024-11-24T08:14:43,269 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/recovered.edits/486.seqid, newMaxSeqId=486, maxSeqId=4 2024-11-24T08:14:43,269 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. 2024-11-24T08:14:43,270 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 0e4dbe9176ee2b005eb8db52027271c6: Waiting for close lock at 1732436081455Running coprocessor pre-close hooks at 1732436081455Disabling compacts and flushes for region at 1732436081455Disabling writes for close at 1732436081456 (+1 ms)Obtaining lock to block concurrent updates at 1732436081456Preparing flush snapshotting stores in 0e4dbe9176ee2b005eb8db52027271c6 at 1732436081456Finished memstore snapshotting TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732436081456Flushing stores of TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6. at 1732436081457 (+1 ms)Flushing 0e4dbe9176ee2b005eb8db52027271c6/A: creating writer at 1732436081457Flushing mob file 0e4dbe9176ee2b005eb8db52027271c6/A: appending metadata at 1732436081464 (+7 ms)Flushing mob file 0e4dbe9176ee2b005eb8db52027271c6/A: closing flushed file at 1732436081464Flushing 0e4dbe9176ee2b005eb8db52027271c6/A: appending metadata at 1732436081876 (+412 ms)Flushing 0e4dbe9176ee2b005eb8db52027271c6/A: closing flushed file at 1732436081876Flushing 0e4dbe9176ee2b005eb8db52027271c6/B: creating writer at 1732436082289 (+413 ms)Flushing 0e4dbe9176ee2b005eb8db52027271c6/B: appending metadata at 1732436082292 (+3 ms)Flushing 0e4dbe9176ee2b005eb8db52027271c6/B: closing flushed file at 1732436082292Flushing 0e4dbe9176ee2b005eb8db52027271c6/C: creating writer at 1732436082703 (+411 ms)Flushing 0e4dbe9176ee2b005eb8db52027271c6/C: appending metadata at 1732436082707 (+4 ms)Flushing 0e4dbe9176ee2b005eb8db52027271c6/C: closing flushed file at 1732436082707Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31a2035b: reopening flushed file at 1732436083118 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@de694f2: reopening flushed file at 1732436083122 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b327b96: reopening flushed file at 1732436083126 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 0e4dbe9176ee2b005eb8db52027271c6 in 1676ms, sequenceid=483, compaction requested=true at 1732436083132 (+6 ms)Writing region close event to WAL at 1732436083265 (+133 ms)Running coprocessor post-close hooks at 1732436083269 (+4 ms)Closed at 1732436083269 2024-11-24T08:14:43,272 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,272 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=0e4dbe9176ee2b005eb8db52027271c6, regionState=CLOSED 2024-11-24T08:14:43,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:14:43,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-24T08:14:43,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 0e4dbe9176ee2b005eb8db52027271c6, server=30c28c82771d,45087,1732436016722 in 1.9710 sec 2024-11-24T08:14:43,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-24T08:14:43,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0e4dbe9176ee2b005eb8db52027271c6, UNASSIGN in 1.9770 sec 2024-11-24T08:14:43,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-24T08:14:43,279 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9800 sec 2024-11-24T08:14:43,281 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436083280"}]},"ts":"1732436083280"} 2024-11-24T08:14:43,282 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T08:14:43,282 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T08:14:43,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9900 sec 2024-11-24T08:14:43,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-24T08:14:43,441 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:43,441 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T08:14:43,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:43,443 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:43,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T08:14:43,443 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=66, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:43,446 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,449 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/recovered.edits] 2024-11-24T08:14:43,453 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/43ccf60eec2447c49e47fed7e1f5790f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/43ccf60eec2447c49e47fed7e1f5790f 2024-11-24T08:14:43,455 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/9e782337e7aa46469c481331ef056b99 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/9e782337e7aa46469c481331ef056b99 2024-11-24T08:14:43,456 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/c90661d1f8674904b1a20f0e9289eb3f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/A/c90661d1f8674904b1a20f0e9289eb3f 2024-11-24T08:14:43,458 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/41297cd0560746a7bdfff2950b90d6b5 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/41297cd0560746a7bdfff2950b90d6b5 2024-11-24T08:14:43,459 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/4e589f7b49b04484bfe6b47108920930 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/4e589f7b49b04484bfe6b47108920930 2024-11-24T08:14:43,461 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/a92b441155e445859d0234b962409761 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/B/a92b441155e445859d0234b962409761 2024-11-24T08:14:43,463 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/ab324b55aa984efa93233afe3f6ff1be to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/ab324b55aa984efa93233afe3f6ff1be 2024-11-24T08:14:43,464 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/c0710971780d4fe7918d6485bdc3b42a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/c0710971780d4fe7918d6485bdc3b42a 2024-11-24T08:14:43,466 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/eb4b6987e63a419b9dc37a449ac775a9 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/C/eb4b6987e63a419b9dc37a449ac775a9 2024-11-24T08:14:43,468 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/recovered.edits/486.seqid to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6/recovered.edits/486.seqid 2024-11-24T08:14:43,469 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,469 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T08:14:43,470 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T08:14:43,471 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T08:14:43,474 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240ac85604b857468c945880f214b0d207_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240ac85604b857468c945880f214b0d207_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,475 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240b3045adaa6c4c6084951fd3d665100c_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240b3045adaa6c4c6084951fd3d665100c_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,477 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112412b024be616b4d88927d72e9fe9acae3_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112412b024be616b4d88927d72e9fe9acae3_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,478 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241e595595237f421ea1db0aa8ad79fa77_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411241e595595237f421ea1db0aa8ad79fa77_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,479 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112425b1d33e0eef4a79bdba63767fc6701c_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112425b1d33e0eef4a79bdba63767fc6701c_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,481 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112437fa5dca58b34c67972618b5c91ef0c1_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112437fa5dca58b34c67972618b5c91ef0c1_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,482 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243c1da917b08b48118aac8c3ae28b0799_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243c1da917b08b48118aac8c3ae28b0799_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,484 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244bd51b6cccfd4cbfbc43f2f0d0115add_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244bd51b6cccfd4cbfbc43f2f0d0115add_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,485 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244e349211cc1444f4984299b81758e621_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244e349211cc1444f4984299b81758e621_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,487 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112466c6481eb1bc4876a372e2dcdbcc1e4a_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112466c6481eb1bc4876a372e2dcdbcc1e4a_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,489 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246c1b25383b5e44c49a14efa895296768_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246c1b25383b5e44c49a14efa895296768_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,490 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112477e932616e524e23b6e2fe208a0248bd_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112477e932616e524e23b6e2fe208a0248bd_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,492 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249507726cf65c418d956c0c5918c07ce2_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249507726cf65c418d956c0c5918c07ce2_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,493 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249d6ffcece2e6497cb6584b5a3f65f936_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249d6ffcece2e6497cb6584b5a3f65f936_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,495 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b4c02828c5bb4bd8bca4d5919bde377f_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b4c02828c5bb4bd8bca4d5919bde377f_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,497 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bde0e807ad9b43dea519f4d28851d40a_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bde0e807ad9b43dea519f4d28851d40a_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,498 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cc3314163ea940669fd7551276613e9b_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cc3314163ea940669fd7551276613e9b_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,500 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d29384ae0f1b41fbbe524160686242ee_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d29384ae0f1b41fbbe524160686242ee_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,501 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e05ee036e9754765a2f04954449fe7de_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e05ee036e9754765a2f04954449fe7de_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,503 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e203ac9536494ccc80f33630ad924c6f_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e203ac9536494ccc80f33630ad924c6f_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,504 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e4f6d283f9f74b6586c24a09a29efa7a_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e4f6d283f9f74b6586c24a09a29efa7a_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,505 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f5e0fb20ef944fb396398a7d05d0bed0_0e4dbe9176ee2b005eb8db52027271c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f5e0fb20ef944fb396398a7d05d0bed0_0e4dbe9176ee2b005eb8db52027271c6 2024-11-24T08:14:43,506 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T08:14:43,508 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=66, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:43,516 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T08:14:43,519 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T08:14:43,520 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=66, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:43,520 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T08:14:43,520 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732436083520"}]},"ts":"9223372036854775807"} 2024-11-24T08:14:43,522 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T08:14:43,522 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0e4dbe9176ee2b005eb8db52027271c6, NAME => 'TestAcidGuarantees,,1732436054577.0e4dbe9176ee2b005eb8db52027271c6.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T08:14:43,522 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T08:14:43,522 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732436083522"}]},"ts":"9223372036854775807"} 2024-11-24T08:14:43,524 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T08:14:43,524 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=66, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:43,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 83 msec 2024-11-24T08:14:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-24T08:14:43,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T08:14:43,550 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:43,560 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-921854773_22 at /127.0.0.1:38396 [Waiting for operation #393] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_554043647_22 at /127.0.0.1:35864 [Waiting for operation #484] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-921854773_22 at /127.0.0.1:47190 [Waiting for operation #272] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/cluster_6ca8f465-b694-0817-15a9-f097eff6822b/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/cluster_6ca8f465-b694-0817-15a9-f097eff6822b/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_554043647_22 at /127.0.0.1:35778 [Waiting for operation #522] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 445) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=383 (was 274) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7414 (was 7487) 2024-11-24T08:14:43,570 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=383, ProcessCount=11, AvailableMemoryMB=7413 2024-11-24T08:14:43,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T08:14:43,572 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:14:43,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T08:14:43,575 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T08:14:43,575 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:43,575 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 67 2024-11-24T08:14:43,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-24T08:14:43,576 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T08:14:43,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742068_1244 (size=1061) 2024-11-24T08:14:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-24T08:14:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-24T08:14:43,988 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6 2024-11-24T08:14:43,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742069_1245 (size=53) 2024-11-24T08:14:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-24T08:14:44,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:14:44,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4e438c4f9c38da2a2238da06f655542f, disabling compactions & flushes 2024-11-24T08:14:44,395 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:44,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:44,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. after waiting 0 ms 2024-11-24T08:14:44,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:44,395 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:44,395 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4e438c4f9c38da2a2238da06f655542f: Waiting for close lock at 1732436084395Disabling compacts and flushes for region at 1732436084395Disabling writes for close at 1732436084395Writing region close event to WAL at 1732436084395Closed at 1732436084395 2024-11-24T08:14:44,396 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T08:14:44,397 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732436084397"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732436084397"}]},"ts":"1732436084397"} 2024-11-24T08:14:44,399 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T08:14:44,400 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T08:14:44,400 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436084400"}]},"ts":"1732436084400"} 2024-11-24T08:14:44,401 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T08:14:44,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e438c4f9c38da2a2238da06f655542f, ASSIGN}] 2024-11-24T08:14:44,403 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e438c4f9c38da2a2238da06f655542f, ASSIGN 2024-11-24T08:14:44,404 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e438c4f9c38da2a2238da06f655542f, ASSIGN; state=OFFLINE, location=30c28c82771d,45087,1732436016722; forceNewPlan=false, retain=false 2024-11-24T08:14:44,555 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=4e438c4f9c38da2a2238da06f655542f, regionState=OPENING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:14:44,557 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e438c4f9c38da2a2238da06f655542f, ASSIGN because future has completed 2024-11-24T08:14:44,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:14:44,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-24T08:14:44,713 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:44,713 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:14:44,713 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:44,714 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:14:44,714 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:44,714 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:44,716 INFO [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:44,718 INFO [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:14:44,718 INFO [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e438c4f9c38da2a2238da06f655542f columnFamilyName A 2024-11-24T08:14:44,718 DEBUG [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:44,723 INFO [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] regionserver.HStore(327): Store=4e438c4f9c38da2a2238da06f655542f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:14:44,723 INFO [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:44,725 INFO [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:14:44,725 INFO [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e438c4f9c38da2a2238da06f655542f columnFamilyName B 2024-11-24T08:14:44,725 DEBUG [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:44,726 INFO [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] regionserver.HStore(327): Store=4e438c4f9c38da2a2238da06f655542f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:14:44,726 INFO [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:44,727 INFO [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:14:44,728 INFO [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e438c4f9c38da2a2238da06f655542f columnFamilyName C 2024-11-24T08:14:44,728 DEBUG [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:14:44,728 INFO [StoreOpener-4e438c4f9c38da2a2238da06f655542f-1 {}] regionserver.HStore(327): Store=4e438c4f9c38da2a2238da06f655542f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:14:44,729 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:44,729 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:44,730 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:44,730 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:44,732 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:44,732 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:44,733 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T08:14:44,734 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:44,739 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:14:44,740 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 4e438c4f9c38da2a2238da06f655542f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67301189, jitterRate=0.0028658658266067505}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T08:14:44,740 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:44,741 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 4e438c4f9c38da2a2238da06f655542f: Running coprocessor pre-open hook at 1732436084714Writing region info on filesystem at 1732436084714Initializing all the Stores at 1732436084715 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436084715Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436084716 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436084716Cleaning up temporary data from old regions at 1732436084732 (+16 ms)Running coprocessor post-open hooks at 1732436084740 (+8 ms)Region opened successfully at 1732436084741 (+1 ms) 2024-11-24T08:14:44,742 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., pid=69, masterSystemTime=1732436084709 2024-11-24T08:14:44,745 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:44,745 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:44,745 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=4e438c4f9c38da2a2238da06f655542f, regionState=OPEN, openSeqNum=2, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:14:44,747 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:14:44,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-24T08:14:44,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 in 195 msec 2024-11-24T08:14:44,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-24T08:14:44,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e438c4f9c38da2a2238da06f655542f, ASSIGN in 353 msec 2024-11-24T08:14:44,758 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T08:14:44,758 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436084758"}]},"ts":"1732436084758"} 2024-11-24T08:14:44,760 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T08:14:44,761 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T08:14:44,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-11-24T08:14:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-24T08:14:45,730 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:45,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e04d2b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,731 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:45,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:45,732 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:45,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:45,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:45,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@255c9231, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:45,733 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52572, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:45,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d1ff44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,734 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:45,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:45,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:45,737 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T08:14:45,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 30c28c82771d,38469,1732436015766 2024-11-24T08:14:45,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@69bc3605 2024-11-24T08:14:45,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T08:14:45,738 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T08:14:45,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,740 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:45,740 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:45,740 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b80ba14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:45,742 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:45,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6acc307b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,743 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:45,743 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:45,744 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@746f89a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:45,745 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:45,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4108ae68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,746 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:45,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:45,747 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60aedef8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:45,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:45,748 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:45,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc7f6af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:45,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:45,749 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5998e316, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:45,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:45,751 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a9b353a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:45,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:45,753 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf771ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:45,754 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:45,754 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f38b8ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:45,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:45,755 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31f5ff39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:45,757 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c2ba027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:45,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:45,760 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:45,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:45,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:45,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6de23898, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:45,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:45,761 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:45,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65865c6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,762 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:45,762 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:45,763 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19caae97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:45,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:45,764 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:45,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3170b92d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:45,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:45,766 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62dff9c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:45,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:45,767 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:45,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f8fbf59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40916ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,768 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:14:45,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:14:45,768 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:14:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:14:45,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:14:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ed4278, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:14:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:14:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:14:45,769 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:14:45,770 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29836bb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:14:45,773 DEBUG [Thread-1146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:45,773 DEBUG [Thread-1155 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:45,773 DEBUG [Thread-1153 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:45,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:45,774 DEBUG [Thread-1151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:45,774 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:45,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:45,775 DEBUG [Thread-1149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:45,775 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:45,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:45,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:45,776 DEBUG [Thread-1157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:45,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:45,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:45,777 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:45,778 DEBUG [Thread-1157 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:45,778 DEBUG [Thread-1157 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:45,779 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:45,779 DEBUG [Thread-1140 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:14:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:14:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-24T08:14:45,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:14:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T08:14:45,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:45,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:14:45,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:45,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:45,783 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:45,785 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:14:45,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:45,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:14:45,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:14:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:45,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:14:45,802 DEBUG [Thread-1138 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:14:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:14:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:14:45,802 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:45,804 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:45,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:14:45,813 DEBUG [Thread-1142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:45,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:45,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:14:45,829 DEBUG [Thread-1144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:14:45,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:14:45,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:14:45,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:14:45,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:14:45,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/1ac59d7499be43a7aabe67cfddc8bd62 is 50, key is test_row_0/A:col10/1732436085794/Put/seqid=0 2024-11-24T08:14:45,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732436085840, max timestamp=1732436085854Num uniques 859; 2024-11-24T08:14:45,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:14:45,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732436085840, max timestamp=1732436085855Num uniques 859; 2024-11-24T08:14:45,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:14:45,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732436085840, max timestamp=1732436085854Num uniques 859; 2024-11-24T08:14:45,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:14:45,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742070_1246 (size=14341) 2024-11-24T08:14:45,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/1ac59d7499be43a7aabe67cfddc8bd62 2024-11-24T08:14:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T08:14:45,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436145885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:45,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:45,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:45,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:45,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436145886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:45,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:45,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436145887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:45,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:45,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436145892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:45,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:45,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436145895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:45,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:45,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/262854abf02e445bbe5285618eb6cc84 is 50, key is test_row_0/B:col10/1732436085794/Put/seqid=0 2024-11-24T08:14:45,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742071_1247 (size=12001) 2024-11-24T08:14:45,928 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/262854abf02e445bbe5285618eb6cc84 2024-11-24T08:14:45,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T08:14:45,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:45,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:45,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:45,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:45,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:45,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/e3f40f2379a146a581f8ebfc010e5b0f is 50, key is test_row_0/C:col10/1732436085794/Put/seqid=0 2024-11-24T08:14:45,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742072_1248 (size=12001) 2024-11-24T08:14:46,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436146000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436146000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436146000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436146000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436146009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T08:14:46,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:46,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:46,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:46,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T08:14:46,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436146210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436146210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436146210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436146210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436146221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T08:14:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:46,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/e3f40f2379a146a581f8ebfc010e5b0f 2024-11-24T08:14:46,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/1ac59d7499be43a7aabe67cfddc8bd62 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/1ac59d7499be43a7aabe67cfddc8bd62 2024-11-24T08:14:46,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/1ac59d7499be43a7aabe67cfddc8bd62, entries=200, sequenceid=12, filesize=14.0 K 2024-11-24T08:14:46,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/262854abf02e445bbe5285618eb6cc84 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/262854abf02e445bbe5285618eb6cc84 2024-11-24T08:14:46,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T08:14:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:46,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/262854abf02e445bbe5285618eb6cc84, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T08:14:46,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/e3f40f2379a146a581f8ebfc010e5b0f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/e3f40f2379a146a581f8ebfc010e5b0f 2024-11-24T08:14:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T08:14:46,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/e3f40f2379a146a581f8ebfc010e5b0f, entries=150, sequenceid=12, filesize=11.7 K 2024-11-24T08:14:46,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4e438c4f9c38da2a2238da06f655542f in 613ms, sequenceid=12, compaction requested=false 2024-11-24T08:14:46,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:46,527 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-24T08:14:46,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:14:46,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:46,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:14:46,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:46,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:14:46,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:46,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/20e80ec208fa4d0b81043eca3667c6c7 is 50, key is test_row_0/A:col10/1732436086526/Put/seqid=0 2024-11-24T08:14:46,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436146530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436146537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436146538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436146540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436146540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742073_1249 (size=12001) 2024-11-24T08:14:46,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T08:14:46,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:46,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:46,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:46,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436146669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436146669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436146669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436146669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436146670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T08:14:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:46,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T08:14:46,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:46,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:46,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:46,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:46,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436146880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436146880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436146880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436146881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:46,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436146881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:46,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:46,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:46,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T08:14:46,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/20e80ec208fa4d0b81043eca3667c6c7 2024-11-24T08:14:46,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/3a631affeeee4d26b870675a67c7d549 is 50, key is test_row_0/B:col10/1732436086526/Put/seqid=0 2024-11-24T08:14:46,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742074_1250 (size=12001) 2024-11-24T08:14:46,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/3a631affeeee4d26b870675a67c7d549 2024-11-24T08:14:46,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/1886bdccc27c48cea91402188ea0a19f is 50, key is test_row_0/C:col10/1732436086526/Put/seqid=0 2024-11-24T08:14:46,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742075_1251 (size=12001) 2024-11-24T08:14:47,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T08:14:47,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:47,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:47,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:47,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:47,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:47,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:47,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T08:14:47,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:47,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:47,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:47,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:47,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:47,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:47,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436147190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436147190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436147190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,195 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436147191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436147191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T08:14:47,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:47,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:47,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:47,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:47,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:47,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/1886bdccc27c48cea91402188ea0a19f 2024-11-24T08:14:47,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/20e80ec208fa4d0b81043eca3667c6c7 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/20e80ec208fa4d0b81043eca3667c6c7 2024-11-24T08:14:47,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/20e80ec208fa4d0b81043eca3667c6c7, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T08:14:47,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/3a631affeeee4d26b870675a67c7d549 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/3a631affeeee4d26b870675a67c7d549 2024-11-24T08:14:47,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/3a631affeeee4d26b870675a67c7d549, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T08:14:47,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/1886bdccc27c48cea91402188ea0a19f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/1886bdccc27c48cea91402188ea0a19f 2024-11-24T08:14:47,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/1886bdccc27c48cea91402188ea0a19f, entries=150, sequenceid=49, filesize=11.7 K 2024-11-24T08:14:47,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 4e438c4f9c38da2a2238da06f655542f in 895ms, sequenceid=49, compaction requested=false 2024-11-24T08:14:47,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:47,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-24T08:14:47,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:47,484 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-24T08:14:47,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:14:47,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:47,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:14:47,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:47,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:14:47,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:47,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/0ae94faafd9f41c688d0004b7c7f576c is 50, key is test_row_0/A:col10/1732436086534/Put/seqid=0 2024-11-24T08:14:47,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742076_1252 (size=12001) 2024-11-24T08:14:47,495 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/0ae94faafd9f41c688d0004b7c7f576c 2024-11-24T08:14:47,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/65027a2805b244a895645c70abfb428d is 50, key is test_row_0/B:col10/1732436086534/Put/seqid=0 2024-11-24T08:14:47,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742077_1253 (size=12001) 2024-11-24T08:14:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:47,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:47,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732436087753, max timestamp=1732436087764Num uniques 859; 2024-11-24T08:14:47,768 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:14:47,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732436087753, max timestamp=1732436087764Num uniques 859; 2024-11-24T08:14:47,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:14:47,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732436087753, max timestamp=1732436087764Num uniques 859; 2024-11-24T08:14:47,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:14:47,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436147794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436147795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436147798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436147800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436147802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436147910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436147910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436147910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436147910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436147910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:47,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T08:14:47,943 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/65027a2805b244a895645c70abfb428d 2024-11-24T08:14:47,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/35c82d91fdeb446c83c09772a7e6aa18 is 50, key is test_row_0/C:col10/1732436086534/Put/seqid=0 2024-11-24T08:14:47,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742078_1254 (size=12001) 2024-11-24T08:14:47,969 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/35c82d91fdeb446c83c09772a7e6aa18 2024-11-24T08:14:47,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/0ae94faafd9f41c688d0004b7c7f576c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/0ae94faafd9f41c688d0004b7c7f576c 2024-11-24T08:14:47,981 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/0ae94faafd9f41c688d0004b7c7f576c, entries=150, sequenceid=58, filesize=11.7 K 2024-11-24T08:14:47,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/65027a2805b244a895645c70abfb428d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/65027a2805b244a895645c70abfb428d 2024-11-24T08:14:47,993 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/65027a2805b244a895645c70abfb428d, entries=150, sequenceid=58, filesize=11.7 K 2024-11-24T08:14:47,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/35c82d91fdeb446c83c09772a7e6aa18 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/35c82d91fdeb446c83c09772a7e6aa18 2024-11-24T08:14:48,001 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/35c82d91fdeb446c83c09772a7e6aa18, entries=150, sequenceid=58, filesize=11.7 K 2024-11-24T08:14:48,002 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 4e438c4f9c38da2a2238da06f655542f in 519ms, sequenceid=58, compaction requested=true 2024-11-24T08:14:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-24T08:14:48,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-24T08:14:48,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-24T08:14:48,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2200 sec 2024-11-24T08:14:48,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.2300 sec 2024-11-24T08:14:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:48,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-24T08:14:48,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:14:48,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T08:14:48,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:14:48,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T08:14:48,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:14:48,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T08:14:48,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/71a854f417c0439b8f92a74747b2cb50 is 50, key is test_row_0/A:col10/1732436087790/Put/seqid=0 2024-11-24T08:14:48,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436148128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436148128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436148128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742079_1255 (size=12001) 2024-11-24T08:14:48,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436148136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436148136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436148240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436148240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436148240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436148250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436148250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436148450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436148451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436148451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436148460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436148470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/71a854f417c0439b8f92a74747b2cb50 2024-11-24T08:14:48,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/2eda52ad1a77466a8632425d0f6165ea is 50, key is test_row_0/B:col10/1732436087790/Put/seqid=0 2024-11-24T08:14:48,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742080_1256 (size=12001) 2024-11-24T08:14:48,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436148760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436148760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436148760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436148770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436148790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:48,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:48,800 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T08:14:49,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/2eda52ad1a77466a8632425d0f6165ea 2024-11-24T08:14:49,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/705de8ccf1bb45619101c77ff260fd50 is 50, key is test_row_0/C:col10/1732436087790/Put/seqid=0 2024-11-24T08:14:49,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742081_1257 (size=12001) 2024-11-24T08:14:49,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436149270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:49,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436149271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:49,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:49,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436149272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:49,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:49,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436149280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:49,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436149300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:49,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:49,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/705de8ccf1bb45619101c77ff260fd50 2024-11-24T08:14:49,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/71a854f417c0439b8f92a74747b2cb50 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/71a854f417c0439b8f92a74747b2cb50 2024-11-24T08:14:49,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/71a854f417c0439b8f92a74747b2cb50, entries=150, sequenceid=97, filesize=11.7 K 2024-11-24T08:14:49,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/2eda52ad1a77466a8632425d0f6165ea as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2eda52ad1a77466a8632425d0f6165ea 2024-11-24T08:14:49,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2eda52ad1a77466a8632425d0f6165ea, entries=150, sequenceid=97, filesize=11.7 K 2024-11-24T08:14:49,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/705de8ccf1bb45619101c77ff260fd50 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/705de8ccf1bb45619101c77ff260fd50 2024-11-24T08:14:49,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/705de8ccf1bb45619101c77ff260fd50, entries=150, sequenceid=97, filesize=11.7 K 2024-11-24T08:14:49,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~440.72 KB/451296, currentSize=33.54 KB/34350 for 4e438c4f9c38da2a2238da06f655542f in 1335ms, sequenceid=97, compaction requested=true 2024-11-24T08:14:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:49,459 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:14:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:49,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:49,459 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:14:49,461 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:14:49,461 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/A is initiating minor compaction (all files) 2024-11-24T08:14:49,461 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:14:49,461 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/B is initiating minor compaction (all files) 2024-11-24T08:14:49,461 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/A in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:49,461 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/B in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:49,461 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/1ac59d7499be43a7aabe67cfddc8bd62, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/20e80ec208fa4d0b81043eca3667c6c7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/0ae94faafd9f41c688d0004b7c7f576c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/71a854f417c0439b8f92a74747b2cb50] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=49.2 K 2024-11-24T08:14:49,461 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/262854abf02e445bbe5285618eb6cc84, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/3a631affeeee4d26b870675a67c7d549, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/65027a2805b244a895645c70abfb428d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2eda52ad1a77466a8632425d0f6165ea] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=46.9 K 2024-11-24T08:14:49,461 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ac59d7499be43a7aabe67cfddc8bd62, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732436085790 2024-11-24T08:14:49,462 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 262854abf02e445bbe5285618eb6cc84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732436085790 2024-11-24T08:14:49,462 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20e80ec208fa4d0b81043eca3667c6c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732436085892 2024-11-24T08:14:49,462 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a631affeeee4d26b870675a67c7d549, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732436085892 2024-11-24T08:14:49,462 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ae94faafd9f41c688d0004b7c7f576c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732436086534 2024-11-24T08:14:49,463 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71a854f417c0439b8f92a74747b2cb50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732436087790 2024-11-24T08:14:49,463 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 65027a2805b244a895645c70abfb428d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732436086534 2024-11-24T08:14:49,463 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eda52ad1a77466a8632425d0f6165ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732436087790 2024-11-24T08:14:49,473 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#A#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:49,473 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/f1160281797a4a1cb34401285f5ad640 is 50, key is test_row_0/A:col10/1732436087790/Put/seqid=0 2024-11-24T08:14:49,475 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#B#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:49,476 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/775293459e53446197f02d303cd04a3a is 50, key is test_row_0/B:col10/1732436087790/Put/seqid=0 2024-11-24T08:14:49,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742083_1259 (size=12139) 2024-11-24T08:14:49,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742082_1258 (size=12139) 2024-11-24T08:14:49,495 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/f1160281797a4a1cb34401285f5ad640 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/f1160281797a4a1cb34401285f5ad640 2024-11-24T08:14:49,501 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/A of 4e438c4f9c38da2a2238da06f655542f into f1160281797a4a1cb34401285f5ad640(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:49,501 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:49,501 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/A, priority=12, startTime=1732436089459; duration=0sec 2024-11-24T08:14:49,501 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:49,501 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:A 2024-11-24T08:14:49,501 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:14:49,502 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:14:49,502 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/C is initiating minor compaction (all files) 2024-11-24T08:14:49,502 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/C in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:49,503 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/e3f40f2379a146a581f8ebfc010e5b0f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/1886bdccc27c48cea91402188ea0a19f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/35c82d91fdeb446c83c09772a7e6aa18, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/705de8ccf1bb45619101c77ff260fd50] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=46.9 K 2024-11-24T08:14:49,503 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3f40f2379a146a581f8ebfc010e5b0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732436085790 2024-11-24T08:14:49,503 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1886bdccc27c48cea91402188ea0a19f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732436085892 2024-11-24T08:14:49,504 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35c82d91fdeb446c83c09772a7e6aa18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732436086534 2024-11-24T08:14:49,504 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 705de8ccf1bb45619101c77ff260fd50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732436087790 2024-11-24T08:14:49,518 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#C#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:49,519 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/6c2bc7eadcb74e52ae34e183e3bc6840 is 50, key is test_row_0/C:col10/1732436087790/Put/seqid=0 2024-11-24T08:14:49,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742084_1260 (size=12139) 2024-11-24T08:14:49,533 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/6c2bc7eadcb74e52ae34e183e3bc6840 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/6c2bc7eadcb74e52ae34e183e3bc6840 2024-11-24T08:14:49,540 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/C of 4e438c4f9c38da2a2238da06f655542f into 6c2bc7eadcb74e52ae34e183e3bc6840(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:49,540 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:49,540 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/C, priority=12, startTime=1732436089459; duration=0sec 2024-11-24T08:14:49,540 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:49,540 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:C 2024-11-24T08:14:49,900 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/775293459e53446197f02d303cd04a3a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/775293459e53446197f02d303cd04a3a 2024-11-24T08:14:49,905 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/B of 4e438c4f9c38da2a2238da06f655542f into 775293459e53446197f02d303cd04a3a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:49,905 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:49,905 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/B, priority=12, startTime=1732436089459; duration=0sec 2024-11-24T08:14:49,905 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:49,905 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:B 2024-11-24T08:14:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-24T08:14:49,940 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:49,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-24T08:14:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T08:14:49,943 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:49,943 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:49,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T08:14:50,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-24T08:14:50,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:50,098 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T08:14:50,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:14:50,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:50,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:14:50,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:50,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:14:50,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:50,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/b7aa96316d334e44a118d5b88851ad52 is 50, key is test_row_0/A:col10/1732436088125/Put/seqid=0 2024-11-24T08:14:50,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742085_1261 (size=9657) 2024-11-24T08:14:50,111 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/b7aa96316d334e44a118d5b88851ad52 2024-11-24T08:14:50,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/dff6ae1021ed41cebcc80cc2245729fd is 50, key is test_row_0/B:col10/1732436088125/Put/seqid=0 2024-11-24T08:14:50,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742086_1262 (size=9657) 2024-11-24T08:14:50,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T08:14:50,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:50,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:50,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436090305, max timestamp=1732436090309Num uniques 859; 2024-11-24T08:14:50,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436090305, max timestamp=1732436090309Num uniques 859; 2024-11-24T08:14:50,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:50,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:50,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732436090305, max timestamp=1732436090309Num uniques 859; 2024-11-24T08:14:50,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-24T08:14:50,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436150335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732436090326, max timestamp=1732436090340Num uniques 1018; 2024-11-24T08:14:50,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T08:14:50,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732436090326, max timestamp=1732436090340Num uniques 1018; 2024-11-24T08:14:50,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732436090326, max timestamp=1732436090340Num uniques 1009; 2024-11-24T08:14:50,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-24T08:14:50,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-24T08:14:50,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732436090358, max timestamp=1732436090364Num uniques 1027; 2024-11-24T08:14:50,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-24T08:14:50,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732436090358, max timestamp=1732436090364Num uniques 1027; 2024-11-24T08:14:50,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732436090358, max timestamp=1732436090364Num uniques 1009; 2024-11-24T08:14:50,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-24T08:14:50,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-24T08:14:50,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436150390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436150391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732436090387, max timestamp=1732436090393Num uniques 1086; 2024-11-24T08:14:50,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-24T08:14:50,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732436090387, max timestamp=1732436090393Num uniques 1086; 2024-11-24T08:14:50,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-24T08:14:50,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1732436090387, max timestamp=1732436090393Num uniques 1086; 2024-11-24T08:14:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-24T08:14:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732436090429, max timestamp=1732436090444Num uniques 1045; 2024-11-24T08:14:50,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732436090429, max timestamp=1732436090444Num uniques 1045; 2024-11-24T08:14:50,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-24T08:14:50,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-24T08:14:50,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732436090429, max timestamp=1732436090444Num uniques 1045; 2024-11-24T08:14:50,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-24T08:14:50,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436150475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436150476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436150481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436150500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436150510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,524 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/dff6ae1021ed41cebcc80cc2245729fd 2024-11-24T08:14:50,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/557052710fe24908bf05359a1a708f6d is 50, key is test_row_0/C:col10/1732436088125/Put/seqid=0 2024-11-24T08:14:50,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742087_1263 (size=9657) 2024-11-24T08:14:50,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T08:14:50,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436150590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436150590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436150590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436150710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436150720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436150800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436150800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436150800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:50,935 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/557052710fe24908bf05359a1a708f6d 2024-11-24T08:14:50,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/b7aa96316d334e44a118d5b88851ad52 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/b7aa96316d334e44a118d5b88851ad52 2024-11-24T08:14:50,945 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/b7aa96316d334e44a118d5b88851ad52, entries=100, sequenceid=108, filesize=9.4 K 2024-11-24T08:14:50,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/dff6ae1021ed41cebcc80cc2245729fd as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/dff6ae1021ed41cebcc80cc2245729fd 2024-11-24T08:14:50,950 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/dff6ae1021ed41cebcc80cc2245729fd, entries=100, sequenceid=108, filesize=9.4 K 2024-11-24T08:14:50,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/557052710fe24908bf05359a1a708f6d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/557052710fe24908bf05359a1a708f6d 2024-11-24T08:14:50,956 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/557052710fe24908bf05359a1a708f6d, entries=100, sequenceid=108, filesize=9.4 K 2024-11-24T08:14:50,957 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for 4e438c4f9c38da2a2238da06f655542f in 859ms, sequenceid=108, compaction requested=false 2024-11-24T08:14:50,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:50,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:50,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-24T08:14:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-24T08:14:50,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-24T08:14:50,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0140 sec 2024-11-24T08:14:50,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0210 sec 2024-11-24T08:14:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:51,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=154.31 KB heapSize=437.39 KB 2024-11-24T08:14:51,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:14:51,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T08:14:51,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:14:51,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T08:14:51,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:14:51,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T08:14:51,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/ffc5bababae246348d863ab5a0dd6b55 is 50, key is test_row_0/A:col10/1732436091022/Put/seqid=0 2024-11-24T08:14:51,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742088_1264 (size=12151) 2024-11-24T08:14:51,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/ffc5bababae246348d863ab5a0dd6b55 2024-11-24T08:14:51,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436151038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436151041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/958b2372ba8b44d1bd434f344d5870ec is 50, key is test_row_0/B:col10/1732436091022/Put/seqid=0 2024-11-24T08:14:51,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742089_1265 (size=12151) 2024-11-24T08:14:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-24T08:14:51,080 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:51,080 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-24T08:14:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T08:14:51,082 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:51,083 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:51,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:51,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436151110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436151110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436151110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436151149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436151149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T08:14:51,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T08:14:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:51,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:51,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436151359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436151359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T08:14:51,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:51,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:51,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:51,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T08:14:51,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/958b2372ba8b44d1bd434f344d5870ec 2024-11-24T08:14:51,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/b9c3f49d70b041c6901a34fb84d6b9cb is 50, key is test_row_0/C:col10/1732436091022/Put/seqid=0 2024-11-24T08:14:51,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742090_1266 (size=12151) 2024-11-24T08:14:51,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/b9c3f49d70b041c6901a34fb84d6b9cb 2024-11-24T08:14:51,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/ffc5bababae246348d863ab5a0dd6b55 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/ffc5bababae246348d863ab5a0dd6b55 2024-11-24T08:14:51,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/ffc5bababae246348d863ab5a0dd6b55, entries=150, sequenceid=215, filesize=11.9 K 2024-11-24T08:14:51,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/958b2372ba8b44d1bd434f344d5870ec as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/958b2372ba8b44d1bd434f344d5870ec 2024-11-24T08:14:51,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/958b2372ba8b44d1bd434f344d5870ec, entries=150, sequenceid=215, filesize=11.9 K 2024-11-24T08:14:51,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/b9c3f49d70b041c6901a34fb84d6b9cb as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/b9c3f49d70b041c6901a34fb84d6b9cb 2024-11-24T08:14:51,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/b9c3f49d70b041c6901a34fb84d6b9cb, entries=150, sequenceid=215, filesize=11.9 K 2024-11-24T08:14:51,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~437.34 KB/447840, currentSize=26.84 KB/27480 for 4e438c4f9c38da2a2238da06f655542f in 459ms, sequenceid=215, compaction requested=true 2024-11-24T08:14:51,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:51,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:51,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:51,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:51,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:51,483 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:51,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:51,483 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:51,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:51,485 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:51,485 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/A is initiating minor compaction (all files) 2024-11-24T08:14:51,486 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/A in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:51,486 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/f1160281797a4a1cb34401285f5ad640, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/b7aa96316d334e44a118d5b88851ad52, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/ffc5bababae246348d863ab5a0dd6b55] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=33.2 K 2024-11-24T08:14:51,486 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:51,486 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/B is initiating minor compaction (all files) 2024-11-24T08:14:51,486 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/B in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:51,486 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/775293459e53446197f02d303cd04a3a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/dff6ae1021ed41cebcc80cc2245729fd, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/958b2372ba8b44d1bd434f344d5870ec] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=33.2 K 2024-11-24T08:14:51,486 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1160281797a4a1cb34401285f5ad640, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732436087790 2024-11-24T08:14:51,486 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 775293459e53446197f02d303cd04a3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732436087790 2024-11-24T08:14:51,486 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7aa96316d334e44a118d5b88851ad52, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732436088125 2024-11-24T08:14:51,487 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting dff6ae1021ed41cebcc80cc2245729fd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732436088125 2024-11-24T08:14:51,487 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffc5bababae246348d863ab5a0dd6b55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732436090473 2024-11-24T08:14:51,487 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 958b2372ba8b44d1bd434f344d5870ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732436090473 2024-11-24T08:14:51,497 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:51,497 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#B#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:51,497 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/52d62f4b89b847338cf10d0c93ad5ffb is 50, key is test_row_0/A:col10/1732436091022/Put/seqid=0 2024-11-24T08:14:51,498 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/f19ae6d4557946c7bf52f923b41346ec is 50, key is test_row_0/B:col10/1732436091022/Put/seqid=0 2024-11-24T08:14:51,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742092_1268 (size=12391) 2024-11-24T08:14:51,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742091_1267 (size=12391) 2024-11-24T08:14:51,514 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/52d62f4b89b847338cf10d0c93ad5ffb as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/52d62f4b89b847338cf10d0c93ad5ffb 2024-11-24T08:14:51,520 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/A of 4e438c4f9c38da2a2238da06f655542f into 52d62f4b89b847338cf10d0c93ad5ffb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:51,520 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:51,520 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/A, priority=13, startTime=1732436091483; duration=0sec 2024-11-24T08:14:51,520 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:51,520 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:A 2024-11-24T08:14:51,521 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:51,522 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:51,522 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/C is initiating minor compaction (all files) 2024-11-24T08:14:51,522 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/C in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:51,522 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/6c2bc7eadcb74e52ae34e183e3bc6840, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/557052710fe24908bf05359a1a708f6d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/b9c3f49d70b041c6901a34fb84d6b9cb] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=33.2 K 2024-11-24T08:14:51,523 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c2bc7eadcb74e52ae34e183e3bc6840, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732436087790 2024-11-24T08:14:51,523 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 557052710fe24908bf05359a1a708f6d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1732436088125 2024-11-24T08:14:51,524 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9c3f49d70b041c6901a34fb84d6b9cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732436090473 2024-11-24T08:14:51,533 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:51,533 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/b5318c7600054d449b51603b5b6d88e3 is 50, key is test_row_0/C:col10/1732436091022/Put/seqid=0 2024-11-24T08:14:51,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742093_1269 (size=12391) 2024-11-24T08:14:51,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-24T08:14:51,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:51,549 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-24T08:14:51,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:14:51,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:51,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:14:51,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:51,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:14:51,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:51,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/6f77986c5a2849e19283d470e2f77544 is 50, key is test_row_0/A:col10/1732436091039/Put/seqid=0 2024-11-24T08:14:51,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742094_1270 (size=9757) 2024-11-24T08:14:51,560 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/b5318c7600054d449b51603b5b6d88e3 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/b5318c7600054d449b51603b5b6d88e3 2024-11-24T08:14:51,561 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/6f77986c5a2849e19283d470e2f77544 2024-11-24T08:14:51,566 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/C of 4e438c4f9c38da2a2238da06f655542f into b5318c7600054d449b51603b5b6d88e3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:51,566 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:51,566 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/C, priority=13, startTime=1732436091483; duration=0sec 2024-11-24T08:14:51,566 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:51,566 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:C 2024-11-24T08:14:51,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/2243a6645d4642188cbfbb735444ec8c is 50, key is test_row_0/B:col10/1732436091039/Put/seqid=0 2024-11-24T08:14:51,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742095_1271 (size=9757) 2024-11-24T08:14:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:51,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:51,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436091632, max timestamp=1732436091652Num uniques 859; 2024-11-24T08:14:51,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:51,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436091632, max timestamp=1732436091652Num uniques 859; 2024-11-24T08:14:51,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:51,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436091632, max timestamp=1732436091652Num uniques 859; 2024-11-24T08:14:51,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:51,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732436091674, max timestamp=1732436091683Num uniques 1018; 2024-11-24T08:14:51,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-24T08:14:51,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732436091674, max timestamp=1732436091683Num uniques 1018; 2024-11-24T08:14:51,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732436091674, max timestamp=1732436091683Num uniques 1018; 2024-11-24T08:14:51,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-24T08:14:51,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-24T08:14:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T08:14:51,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1732436091701, max timestamp=1732436091712Num uniques 1127; 2024-11-24T08:14:51,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=17262, suffix heap size=145176,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=377 2024-11-24T08:14:51,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732436091701, max timestamp=1732436091712Num uniques 1127; 2024-11-24T08:14:51,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-24T08:14:51,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1732436091701, max timestamp=1732436091712Num uniques 1127; 2024-11-24T08:14:51,719 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=17262, suffix heap size=145176,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=377 2024-11-24T08:14:51,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436151740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436151740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436151746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1732436091738, max timestamp=1732436091745Num uniques 1236; 2024-11-24T08:14:51,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=56596, new segment data size=8506, suffix heap size=156512,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=186 2024-11-24T08:14:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1732436091738, max timestamp=1732436091745Num uniques 1036; 2024-11-24T08:14:51,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47436, new segment data size=8506, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-24T08:14:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8506, totalHeapSize=19584, min timestamp=1732436091738, max timestamp=1732436091745Num uniques 1236; 2024-11-24T08:14:51,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=56596, new segment data size=8506, suffix heap size=156512,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1236, new segment cells count=186 2024-11-24T08:14:51,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732436091798, max timestamp=1732436091816Num uniques 1045; 2024-11-24T08:14:51,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-24T08:14:51,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732436091798, max timestamp=1732436091816Num uniques 1045; 2024-11-24T08:14:51,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-24T08:14:51,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732436091798, max timestamp=1732436091816Num uniques 1045; 2024-11-24T08:14:51,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-24T08:14:51,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436151860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436151860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:51,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732436091835, max timestamp=1732436091864Num uniques 1104; 2024-11-24T08:14:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436151862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:51,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732436091835, max timestamp=1732436091864Num uniques 1104; 2024-11-24T08:14:51,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-24T08:14:51,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-24T08:14:51,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732436091835, max timestamp=1732436091864Num uniques 1104; 2024-11-24T08:14:51,869 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-24T08:14:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:51,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:51,908 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/f19ae6d4557946c7bf52f923b41346ec as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f19ae6d4557946c7bf52f923b41346ec 2024-11-24T08:14:51,912 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/B of 4e438c4f9c38da2a2238da06f655542f into f19ae6d4557946c7bf52f923b41346ec(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:51,912 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:51,912 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/B, priority=13, startTime=1732436091483; duration=0sec 2024-11-24T08:14:51,912 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:51,912 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:B 2024-11-24T08:14:51,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732436091913, max timestamp=1732436091927Num uniques 1063; 2024-11-24T08:14:51,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-24T08:14:51,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732436091913, max timestamp=1732436091927Num uniques 1063; 2024-11-24T08:14:51,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-24T08:14:51,932 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732436091913, max timestamp=1732436091927Num uniques 1063; 2024-11-24T08:14:51,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-24T08:14:51,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732436091950, max timestamp=1732436091973Num uniques 1022; 2024-11-24T08:14:51,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-24T08:14:51,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732436091950, max timestamp=1732436091973Num uniques 1022; 2024-11-24T08:14:51,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732436091950, max timestamp=1732436091973Num uniques 1022; 2024-11-24T08:14:51,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-24T08:14:51,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-24T08:14:51,979 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/2243a6645d4642188cbfbb735444ec8c 2024-11-24T08:14:51,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/a94240476afe403ca6ec3134434fbab8 is 50, key is test_row_0/C:col10/1732436091039/Put/seqid=0 2024-11-24T08:14:51,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742096_1272 (size=9757) 2024-11-24T08:14:52,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732436091999, max timestamp=1732436092007Num uniques 1031; 2024-11-24T08:14:52,012 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-24T08:14:52,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732436091999, max timestamp=1732436092007Num uniques 1031; 2024-11-24T08:14:52,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-24T08:14:52,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1732436091999, max timestamp=1732436092007Num uniques 1031; 2024-11-24T08:14:52,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-24T08:14:52,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436152039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732436092018, max timestamp=1732436092042Num uniques 1090; 2024-11-24T08:14:52,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-24T08:14:52,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732436092018, max timestamp=1732436092042Num uniques 1090; 2024-11-24T08:14:52,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732436092018, max timestamp=1732436092042Num uniques 1090; 2024-11-24T08:14:52,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-24T08:14:52,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-11-24T08:14:52,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1732436092067, max timestamp=1732436092077Num uniques 1099; 2024-11-24T08:14:52,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=15984, suffix heap size=142264,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=349 2024-11-24T08:14:52,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1732436092067, max timestamp=1732436092077Num uniques 1099; 2024-11-24T08:14:52,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=15984, suffix heap size=142264,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=349 2024-11-24T08:14:52,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=349, cellSize=15984, totalHeapSize=36536, min timestamp=1732436092067, max timestamp=1732436092077Num uniques 1099; 2024-11-24T08:14:52,082 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=15984, suffix heap size=142264,new segment heap size=36536 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=349 2024-11-24T08:14:52,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436152106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11808, totalHeapSize=27072, min timestamp=1732436092092, max timestamp=1732436092112Num uniques 1208; 2024-11-24T08:14:52,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11808, totalHeapSize=27072, min timestamp=1732436092092, max timestamp=1732436092112Num uniques 1208; 2024-11-24T08:14:52,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732436092092, max timestamp=1732436092112Num uniques 1208; 2024-11-24T08:14:52,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=55318, new segment data size=11808, suffix heap size=153600,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=258 2024-11-24T08:14:52,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=55318, new segment data size=9978, suffix heap size=153600,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=218 2024-11-24T08:14:52,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=55318, new segment data size=11808, suffix heap size=153600,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1208, new segment cells count=258 2024-11-24T08:14:52,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436152167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436152168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1732436092159, max timestamp=1732436092173Num uniques 1117; 2024-11-24T08:14:52,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=7642, suffix heap size=144136,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=167 2024-11-24T08:14:52,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1732436092159, max timestamp=1732436092173Num uniques 1077; 2024-11-24T08:14:52,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=7642, suffix heap size=139976,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=167 2024-11-24T08:14:52,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1732436092159, max timestamp=1732436092173Num uniques 1117; 2024-11-24T08:14:52,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=7642, suffix heap size=144136,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=167 2024-11-24T08:14:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,184 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T08:14:52,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1732436092229, max timestamp=1732436092241Num uniques 1026; 2024-11-24T08:14:52,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-24T08:14:52,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1732436092229, max timestamp=1732436092241Num uniques 1026; 2024-11-24T08:14:52,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1732436092229, max timestamp=1732436092241Num uniques 1026; 2024-11-24T08:14:52,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-24T08:14:52,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-24T08:14:52,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732436092266, max timestamp=1732436092279Num uniques 1035; 2024-11-24T08:14:52,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-11-24T08:14:52,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732436092266, max timestamp=1732436092279Num uniques 1035; 2024-11-24T08:14:52,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1732436092266, max timestamp=1732436092279Num uniques 1035; 2024-11-24T08:14:52,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-11-24T08:14:52,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-11-24T08:14:52,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436152304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436152305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436152306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1732436092295, max timestamp=1732436092306Num uniques 1094; 2024-11-24T08:14:52,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=13464, suffix heap size=141744,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=294 2024-11-24T08:14:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1732436092295, max timestamp=1732436092306Num uniques 1094; 2024-11-24T08:14:52,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=13464, suffix heap size=141744,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=294 2024-11-24T08:14:52,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=294, cellSize=13464, totalHeapSize=30816, min timestamp=1732436092295, max timestamp=1732436092306Num uniques 1094; 2024-11-24T08:14:52,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=13464, suffix heap size=141744,new segment heap size=30816 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=294 2024-11-24T08:14:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436152351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436152354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,391 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/a94240476afe403ca6ec3134434fbab8 2024-11-24T08:14:52,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/6f77986c5a2849e19283d470e2f77544 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/6f77986c5a2849e19283d470e2f77544 2024-11-24T08:14:52,399 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/6f77986c5a2849e19283d470e2f77544, entries=100, sequenceid=223, filesize=9.5 K 2024-11-24T08:14:52,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/2243a6645d4642188cbfbb735444ec8c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2243a6645d4642188cbfbb735444ec8c 2024-11-24T08:14:52,404 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2243a6645d4642188cbfbb735444ec8c, entries=100, sequenceid=223, filesize=9.5 K 2024-11-24T08:14:52,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/a94240476afe403ca6ec3134434fbab8 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/a94240476afe403ca6ec3134434fbab8 2024-11-24T08:14:52,411 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/a94240476afe403ca6ec3134434fbab8, entries=100, sequenceid=223, filesize=9.5 K 2024-11-24T08:14:52,412 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=154.31 KB/158010 for 4e438c4f9c38da2a2238da06f655542f in 863ms, sequenceid=223, compaction requested=false 2024-11-24T08:14:52,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:52,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:52,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-24T08:14:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-24T08:14:52,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-24T08:14:52,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3320 sec 2024-11-24T08:14:52,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.3370 sec 2024-11-24T08:14:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:52,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=161.02 KB heapSize=452.72 KB 2024-11-24T08:14:52,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:14:52,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T08:14:52,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:14:52,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T08:14:52,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:14:52,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-24T08:14:52,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/798d1fd5bcee426cbc5ea0bc5c572dc2 is 50, key is test_row_0/A:col10/1732436092423/Put/seqid=0 2024-11-24T08:14:52,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436152430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436152430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436152439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742097_1273 (size=14741) 2024-11-24T08:14:52,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436152460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436152469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436152540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436152540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436152550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436152670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436152679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436152750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436152750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436152759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/798d1fd5bcee426cbc5ea0bc5c572dc2 2024-11-24T08:14:52,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/e4df57be80ca411b88e49844fcf21c68 is 50, key is test_row_0/B:col10/1732436092423/Put/seqid=0 2024-11-24T08:14:52,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742098_1274 (size=12301) 2024-11-24T08:14:52,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436152980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:52,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436152989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:53,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436153060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:53,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436153060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:53,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:53,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:53,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436153069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:53,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-24T08:14:53,230 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:53,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:53,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-24T08:14:53,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T08:14:53,232 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:53,233 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:53,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:53,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/e4df57be80ca411b88e49844fcf21c68 2024-11-24T08:14:53,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/9898ee77872b4642b80f3534d949dd31 is 50, key is test_row_0/C:col10/1732436092423/Put/seqid=0 2024-11-24T08:14:53,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742099_1275 (size=12301) 2024-11-24T08:14:53,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T08:14:53,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T08:14:53,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:53,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:53,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:53,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:53,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:53,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:53,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436153490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:53,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:53,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436153500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,503 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:53,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T08:14:53,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:53,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:53,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:53,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:53,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:53,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T08:14:53,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436153580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:53,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:53,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436153580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:53,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:53,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436153580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:53,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:53,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:53,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/9898ee77872b4642b80f3534d949dd31 2024-11-24T08:14:53,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/798d1fd5bcee426cbc5ea0bc5c572dc2 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/798d1fd5bcee426cbc5ea0bc5c572dc2 2024-11-24T08:14:53,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/798d1fd5bcee426cbc5ea0bc5c572dc2, entries=200, sequenceid=521, filesize=14.4 K 2024-11-24T08:14:53,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/e4df57be80ca411b88e49844fcf21c68 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/e4df57be80ca411b88e49844fcf21c68 2024-11-24T08:14:53,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/e4df57be80ca411b88e49844fcf21c68, entries=150, sequenceid=521, filesize=12.0 K 2024-11-24T08:14:53,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/9898ee77872b4642b80f3534d949dd31 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/9898ee77872b4642b80f3534d949dd31 2024-11-24T08:14:53,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T08:14:53,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:53,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:53,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:53,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:53,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:53,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:53,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/9898ee77872b4642b80f3534d949dd31, entries=150, sequenceid=521, filesize=12.0 K 2024-11-24T08:14:53,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~452.67 KB/463536, currentSize=20.13 KB/20610 for 4e438c4f9c38da2a2238da06f655542f in 1271ms, sequenceid=521, compaction requested=true 2024-11-24T08:14:53,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:53,695 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:53,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:53,696 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:53,697 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:53,698 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/A is initiating minor compaction (all files) 2024-11-24T08:14:53,698 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/A in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:53,698 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/52d62f4b89b847338cf10d0c93ad5ffb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/6f77986c5a2849e19283d470e2f77544, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/798d1fd5bcee426cbc5ea0bc5c572dc2] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.0 K 2024-11-24T08:14:53,699 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:53,699 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/B is initiating minor compaction (all files) 2024-11-24T08:14:53,699 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/B in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:53,700 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f19ae6d4557946c7bf52f923b41346ec, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2243a6645d4642188cbfbb735444ec8c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/e4df57be80ca411b88e49844fcf21c68] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=33.6 K 2024-11-24T08:14:53,700 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52d62f4b89b847338cf10d0c93ad5ffb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732436090473 2024-11-24T08:14:53,700 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting f19ae6d4557946c7bf52f923b41346ec, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732436090473 2024-11-24T08:14:53,700 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f77986c5a2849e19283d470e2f77544, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732436091034 2024-11-24T08:14:53,701 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2243a6645d4642188cbfbb735444ec8c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732436091034 2024-11-24T08:14:53,701 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting e4df57be80ca411b88e49844fcf21c68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732436092347 2024-11-24T08:14:53,702 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 798d1fd5bcee426cbc5ea0bc5c572dc2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732436092347 2024-11-24T08:14:53,714 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#B#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:53,715 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/21adf159b9434e95ace50d988a011cfa is 50, key is test_row_0/B:col10/1732436092423/Put/seqid=0 2024-11-24T08:14:53,718 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#A#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:53,718 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/ad9e7b19e27d4afcae0ae99b3dec8de6 is 50, key is test_row_0/A:col10/1732436092423/Put/seqid=0 2024-11-24T08:14:53,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742100_1276 (size=12643) 2024-11-24T08:14:53,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742101_1277 (size=12643) 2024-11-24T08:14:53,731 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/21adf159b9434e95ace50d988a011cfa as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/21adf159b9434e95ace50d988a011cfa 2024-11-24T08:14:53,737 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/ad9e7b19e27d4afcae0ae99b3dec8de6 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/ad9e7b19e27d4afcae0ae99b3dec8de6 2024-11-24T08:14:53,738 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/B of 4e438c4f9c38da2a2238da06f655542f into 21adf159b9434e95ace50d988a011cfa(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:53,738 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:53,738 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/B, priority=13, startTime=1732436093696; duration=0sec 2024-11-24T08:14:53,738 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:53,738 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:B 2024-11-24T08:14:53,739 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:53,740 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:53,740 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/C is initiating minor compaction (all files) 2024-11-24T08:14:53,740 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/C in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:53,740 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/b5318c7600054d449b51603b5b6d88e3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/a94240476afe403ca6ec3134434fbab8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/9898ee77872b4642b80f3534d949dd31] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=33.6 K 2024-11-24T08:14:53,740 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting b5318c7600054d449b51603b5b6d88e3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1732436090473 2024-11-24T08:14:53,741 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting a94240476afe403ca6ec3134434fbab8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732436091034 2024-11-24T08:14:53,741 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9898ee77872b4642b80f3534d949dd31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732436092347 2024-11-24T08:14:53,744 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/A of 4e438c4f9c38da2a2238da06f655542f into ad9e7b19e27d4afcae0ae99b3dec8de6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:53,744 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:53,744 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/A, priority=13, startTime=1732436093695; duration=0sec 2024-11-24T08:14:53,745 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:53,745 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:A 2024-11-24T08:14:53,751 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:53,752 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/16b96be86774407e95f0acd91705c0e1 is 50, key is test_row_0/C:col10/1732436092423/Put/seqid=0 2024-11-24T08:14:53,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742102_1278 (size=12643) 2024-11-24T08:14:53,765 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/16b96be86774407e95f0acd91705c0e1 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/16b96be86774407e95f0acd91705c0e1 2024-11-24T08:14:53,771 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/C of 4e438c4f9c38da2a2238da06f655542f into 16b96be86774407e95f0acd91705c0e1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:53,771 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:53,771 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/C, priority=13, startTime=1732436093696; duration=0sec 2024-11-24T08:14:53,771 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:53,772 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:C 2024-11-24T08:14:53,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-24T08:14:53,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:53,846 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-24T08:14:53,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:14:53,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:53,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:14:53,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:53,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:14:53,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:53,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/cb204720cfe246dd961354905169d99e is 50, key is test_row_0/A:col10/1732436092429/Put/seqid=0 2024-11-24T08:14:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T08:14:53,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742103_1279 (size=9857) 2024-11-24T08:14:53,863 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/cb204720cfe246dd961354905169d99e 2024-11-24T08:14:53,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/e60b8abcb3cb4e59b0c80cb1c57b5303 is 50, key is test_row_0/B:col10/1732436092429/Put/seqid=0 2024-11-24T08:14:53,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742104_1280 (size=9857) 2024-11-24T08:14:54,279 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/e60b8abcb3cb4e59b0c80cb1c57b5303 2024-11-24T08:14:54,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/de88a25d875845e8bd7206b5a7989271 is 50, key is test_row_0/C:col10/1732436092429/Put/seqid=0 2024-11-24T08:14:54,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742105_1281 (size=9857) 2024-11-24T08:14:54,302 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=530 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/de88a25d875845e8bd7206b5a7989271 2024-11-24T08:14:54,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/cb204720cfe246dd961354905169d99e as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/cb204720cfe246dd961354905169d99e 2024-11-24T08:14:54,314 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/cb204720cfe246dd961354905169d99e, entries=100, sequenceid=530, filesize=9.6 K 2024-11-24T08:14:54,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/e60b8abcb3cb4e59b0c80cb1c57b5303 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/e60b8abcb3cb4e59b0c80cb1c57b5303 2024-11-24T08:14:54,319 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/e60b8abcb3cb4e59b0c80cb1c57b5303, entries=100, sequenceid=530, filesize=9.6 K 2024-11-24T08:14:54,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/de88a25d875845e8bd7206b5a7989271 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/de88a25d875845e8bd7206b5a7989271 2024-11-24T08:14:54,326 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/de88a25d875845e8bd7206b5a7989271, entries=100, sequenceid=530, filesize=9.6 K 2024-11-24T08:14:54,327 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 4e438c4f9c38da2a2238da06f655542f in 481ms, sequenceid=530, compaction requested=false 2024-11-24T08:14:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-24T08:14:54,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-24T08:14:54,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-24T08:14:54,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0960 sec 2024-11-24T08:14:54,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.1020 sec 2024-11-24T08:14:54,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-24T08:14:54,370 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:54,371 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-24T08:14:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T08:14:54,373 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:54,374 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:54,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T08:14:54,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:54,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:14:54,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:14:54,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:54,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:14:54,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:54,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:14:54,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:54,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/299edc49114646f39d7cce66d231817e is 50, key is test_row_0/A:col10/1732436094515/Put/seqid=0 2024-11-24T08:14:54,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T08:14:54,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:54,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:54,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:54,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:54,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:54,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:54,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436154562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436094550, max timestamp=1732436094565Num uniques 859; 2024-11-24T08:14:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436094550, max timestamp=1732436094565Num uniques 859; 2024-11-24T08:14:54,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:54,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:54,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436094550, max timestamp=1732436094565Num uniques 859; 2024-11-24T08:14:54,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:14:54,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742106_1282 (size=14741) 2024-11-24T08:14:54,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436154617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436154618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436154619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436154624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436154680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T08:14:54,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T08:14:54,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:54,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:54,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:54,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:54,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:54,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436154730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436154730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436154730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436154740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T08:14:54,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:54,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:54,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436154890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436154940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436154940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436154940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436154950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:54,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/299edc49114646f39d7cce66d231817e 2024-11-24T08:14:54,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/99f01cf9f17744abb3d4b4ff50d24c9c is 50, key is test_row_0/B:col10/1732436094515/Put/seqid=0 2024-11-24T08:14:54,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742107_1283 (size=12301) 2024-11-24T08:14:54,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/99f01cf9f17744abb3d4b4ff50d24c9c 2024-11-24T08:14:54,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/7c26f67b25ce4621b2cc67c6bc8f00e4 is 50, key is test_row_0/C:col10/1732436094515/Put/seqid=0 2024-11-24T08:14:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T08:14:55,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742108_1284 (size=12301) 2024-11-24T08:14:55,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T08:14:55,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:55,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:55,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:55,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:55,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:55,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:55,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T08:14:55,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:55,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:55,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:55,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:55,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:55,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436155200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:55,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436155250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:55,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436155250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:55,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,254 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436155250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:55,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:55,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436155260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:55,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T08:14:55,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:55,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:55,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:55,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:55,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:14:55,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/7c26f67b25ce4621b2cc67c6bc8f00e4 2024-11-24T08:14:55,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/299edc49114646f39d7cce66d231817e as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/299edc49114646f39d7cce66d231817e 2024-11-24T08:14:55,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/299edc49114646f39d7cce66d231817e, entries=200, sequenceid=541, filesize=14.4 K 2024-11-24T08:14:55,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/99f01cf9f17744abb3d4b4ff50d24c9c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/99f01cf9f17744abb3d4b4ff50d24c9c 2024-11-24T08:14:55,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/99f01cf9f17744abb3d4b4ff50d24c9c, entries=150, sequenceid=541, filesize=12.0 K 2024-11-24T08:14:55,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/7c26f67b25ce4621b2cc67c6bc8f00e4 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/7c26f67b25ce4621b2cc67c6bc8f00e4 2024-11-24T08:14:55,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/7c26f67b25ce4621b2cc67c6bc8f00e4, entries=150, sequenceid=541, filesize=12.0 K 2024-11-24T08:14:55,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4e438c4f9c38da2a2238da06f655542f in 898ms, sequenceid=541, compaction requested=true 2024-11-24T08:14:55,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:55,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:55,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:55,418 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:55,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:55,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:55,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:55,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-24T08:14:55,418 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:55,419 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37241 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:55,419 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/A is initiating minor compaction (all files) 2024-11-24T08:14:55,419 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/A in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:55,419 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/ad9e7b19e27d4afcae0ae99b3dec8de6, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/cb204720cfe246dd961354905169d99e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/299edc49114646f39d7cce66d231817e] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.4 K 2024-11-24T08:14:55,419 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:55,419 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/B is initiating minor compaction (all files) 2024-11-24T08:14:55,419 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/B in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:55,419 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/21adf159b9434e95ace50d988a011cfa, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/e60b8abcb3cb4e59b0c80cb1c57b5303, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/99f01cf9f17744abb3d4b4ff50d24c9c] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=34.0 K 2024-11-24T08:14:55,420 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad9e7b19e27d4afcae0ae99b3dec8de6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732436092347 2024-11-24T08:14:55,420 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 21adf159b9434e95ace50d988a011cfa, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732436092347 2024-11-24T08:14:55,420 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb204720cfe246dd961354905169d99e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732436092427 2024-11-24T08:14:55,420 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting e60b8abcb3cb4e59b0c80cb1c57b5303, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732436092427 2024-11-24T08:14:55,421 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 299edc49114646f39d7cce66d231817e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732436094512 2024-11-24T08:14:55,421 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 99f01cf9f17744abb3d4b4ff50d24c9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732436094512 2024-11-24T08:14:55,427 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#A#compaction#237 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:55,428 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/00b38a4025964b51a099a6e920414850 is 50, key is test_row_0/A:col10/1732436094515/Put/seqid=0 2024-11-24T08:14:55,430 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:55,431 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/60a640c14ff3454a90f37064ad5ee9ef is 50, key is test_row_0/B:col10/1732436094515/Put/seqid=0 2024-11-24T08:14:55,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742109_1285 (size=12745) 2024-11-24T08:14:55,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742110_1286 (size=12745) 2024-11-24T08:14:55,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-24T08:14:55,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:55,462 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-24T08:14:55,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:14:55,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:55,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:14:55,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:55,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:14:55,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:14:55,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/395cb25699bd4e2eb9db512538db64f2 is 50, key is test_row_0/A:col10/1732436094621/Put/seqid=0 2024-11-24T08:14:55,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742111_1287 (size=12301) 2024-11-24T08:14:55,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T08:14:55,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:55,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:55,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:55,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436155754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:55,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436155760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436155760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:55,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:55,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436155769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:55,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436155769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:55,846 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/00b38a4025964b51a099a6e920414850 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/00b38a4025964b51a099a6e920414850 2024-11-24T08:14:55,846 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/60a640c14ff3454a90f37064ad5ee9ef as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/60a640c14ff3454a90f37064ad5ee9ef 2024-11-24T08:14:55,852 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/A of 4e438c4f9c38da2a2238da06f655542f into 00b38a4025964b51a099a6e920414850(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:55,852 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:55,852 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/A, priority=13, startTime=1732436095418; duration=0sec 2024-11-24T08:14:55,852 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/B of 4e438c4f9c38da2a2238da06f655542f into 60a640c14ff3454a90f37064ad5ee9ef(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:55,852 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:55,852 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/B, priority=13, startTime=1732436095418; duration=0sec 2024-11-24T08:14:55,852 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:55,852 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:A 2024-11-24T08:14:55,853 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:55,853 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:55,853 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:B 2024-11-24T08:14:55,853 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:55,853 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/C is initiating minor compaction (all files) 2024-11-24T08:14:55,853 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/C in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:55,853 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/16b96be86774407e95f0acd91705c0e1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/de88a25d875845e8bd7206b5a7989271, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/7c26f67b25ce4621b2cc67c6bc8f00e4] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=34.0 K 2024-11-24T08:14:55,854 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16b96be86774407e95f0acd91705c0e1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1732436092347 2024-11-24T08:14:55,854 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting de88a25d875845e8bd7206b5a7989271, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=530, earliestPutTs=1732436092427 2024-11-24T08:14:55,854 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c26f67b25ce4621b2cc67c6bc8f00e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732436094512 2024-11-24T08:14:55,861 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#C#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:55,862 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/f6af2ae716fd4e6db73326800f690228 is 50, key is test_row_0/C:col10/1732436094515/Put/seqid=0 2024-11-24T08:14:55,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742112_1288 (size=12745) 2024-11-24T08:14:55,872 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/395cb25699bd4e2eb9db512538db64f2 2024-11-24T08:14:55,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:55,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436155869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:55,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:55,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/fd910f38e44741ed9890a83142ae0ba0 is 50, key is test_row_0/B:col10/1732436094621/Put/seqid=0 2024-11-24T08:14:55,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742113_1289 (size=12301) 2024-11-24T08:14:56,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436156080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:56,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:56,271 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/f6af2ae716fd4e6db73326800f690228 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/f6af2ae716fd4e6db73326800f690228 2024-11-24T08:14:56,276 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/C of 4e438c4f9c38da2a2238da06f655542f into f6af2ae716fd4e6db73326800f690228(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:56,276 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:56,276 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/C, priority=13, startTime=1732436095418; duration=0sec 2024-11-24T08:14:56,277 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:56,277 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:C 2024-11-24T08:14:56,292 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/fd910f38e44741ed9890a83142ae0ba0 2024-11-24T08:14:56,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/070fe0ea694e4d9cb1c08a5644d4dbe5 is 50, key is test_row_0/C:col10/1732436094621/Put/seqid=0 2024-11-24T08:14:56,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742114_1290 (size=12301) 2024-11-24T08:14:56,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436156390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:56,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T08:14:56,703 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=577 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/070fe0ea694e4d9cb1c08a5644d4dbe5 2024-11-24T08:14:56,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/395cb25699bd4e2eb9db512538db64f2 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/395cb25699bd4e2eb9db512538db64f2 2024-11-24T08:14:56,712 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/395cb25699bd4e2eb9db512538db64f2, entries=150, sequenceid=577, filesize=12.0 K 2024-11-24T08:14:56,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/fd910f38e44741ed9890a83142ae0ba0 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/fd910f38e44741ed9890a83142ae0ba0 2024-11-24T08:14:56,716 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/fd910f38e44741ed9890a83142ae0ba0, entries=150, sequenceid=577, filesize=12.0 K 2024-11-24T08:14:56,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/070fe0ea694e4d9cb1c08a5644d4dbe5 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/070fe0ea694e4d9cb1c08a5644d4dbe5 2024-11-24T08:14:56,721 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/070fe0ea694e4d9cb1c08a5644d4dbe5, entries=150, sequenceid=577, filesize=12.0 K 2024-11-24T08:14:56,722 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 4e438c4f9c38da2a2238da06f655542f in 1260ms, sequenceid=577, compaction requested=false 2024-11-24T08:14:56,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:56,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:56,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-24T08:14:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-24T08:14:56,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-24T08:14:56,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3490 sec 2024-11-24T08:14:56,727 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 2.3550 sec 2024-11-24T08:14:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:56,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T08:14:56,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:14:56,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:56,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:14:56,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:56,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:14:56,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:56,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/e59f006146cf4c81a3ab2795ab2b3074 is 50, key is test_row_0/A:col10/1732436095738/Put/seqid=0 2024-11-24T08:14:56,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742115_1291 (size=12301) 2024-11-24T08:14:56,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436156809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:56,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436156810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:56,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436156814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:56,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436156816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:56,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436156900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:56,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436156920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:56,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436156920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:56,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:56,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436156930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:56,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436156930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:56,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:57,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436157130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:57,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436157130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:57,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436157140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:57,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436157140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:57,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/e59f006146cf4c81a3ab2795ab2b3074 2024-11-24T08:14:57,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/b697b04ad18b4baa9debb3b9e084a266 is 50, key is test_row_0/B:col10/1732436095738/Put/seqid=0 2024-11-24T08:14:57,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742116_1292 (size=12301) 2024-11-24T08:14:57,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436157440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:57,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436157440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:57,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436157450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:57,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:57,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436157450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:57,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/b697b04ad18b4baa9debb3b9e084a266 2024-11-24T08:14:57,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/e060bab8704e47ff8f38d3d62cd4e339 is 50, key is test_row_0/C:col10/1732436095738/Put/seqid=0 2024-11-24T08:14:57,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742117_1293 (size=12301) 2024-11-24T08:14:57,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:57,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436157910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:57,914 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in 4e438c4f9c38da2a2238da06f655542f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:14:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:57,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436157950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:57,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:57,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436157950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:57,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:57,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436157960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:57,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436157960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:57,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:58,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/e060bab8704e47ff8f38d3d62cd4e339 2024-11-24T08:14:58,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/e59f006146cf4c81a3ab2795ab2b3074 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/e59f006146cf4c81a3ab2795ab2b3074 2024-11-24T08:14:58,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/e59f006146cf4c81a3ab2795ab2b3074, entries=150, sequenceid=592, filesize=12.0 K 2024-11-24T08:14:58,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/b697b04ad18b4baa9debb3b9e084a266 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/b697b04ad18b4baa9debb3b9e084a266 2024-11-24T08:14:58,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/b697b04ad18b4baa9debb3b9e084a266, entries=150, sequenceid=592, filesize=12.0 K 2024-11-24T08:14:58,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/e060bab8704e47ff8f38d3d62cd4e339 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/e060bab8704e47ff8f38d3d62cd4e339 2024-11-24T08:14:58,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/e060bab8704e47ff8f38d3d62cd4e339, entries=150, sequenceid=592, filesize=12.0 K 2024-11-24T08:14:58,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4e438c4f9c38da2a2238da06f655542f in 1258ms, sequenceid=592, compaction requested=true 2024-11-24T08:14:58,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:58,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:14:58,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:58,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:14:58,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:58,031 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:58,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:14:58,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:58,031 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:58,032 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:58,032 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/A is initiating minor compaction (all files) 2024-11-24T08:14:58,032 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/A in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:58,032 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/00b38a4025964b51a099a6e920414850, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/395cb25699bd4e2eb9db512538db64f2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/e59f006146cf4c81a3ab2795ab2b3074] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.5 K 2024-11-24T08:14:58,033 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:58,033 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00b38a4025964b51a099a6e920414850, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732436094512 2024-11-24T08:14:58,033 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/B is initiating minor compaction (all files) 2024-11-24T08:14:58,033 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/B in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:58,033 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/60a640c14ff3454a90f37064ad5ee9ef, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/fd910f38e44741ed9890a83142ae0ba0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/b697b04ad18b4baa9debb3b9e084a266] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.5 K 2024-11-24T08:14:58,033 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 395cb25699bd4e2eb9db512538db64f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1732436094605 2024-11-24T08:14:58,033 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 60a640c14ff3454a90f37064ad5ee9ef, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732436094512 2024-11-24T08:14:58,034 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e59f006146cf4c81a3ab2795ab2b3074, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1732436095738 2024-11-24T08:14:58,034 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting fd910f38e44741ed9890a83142ae0ba0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1732436094605 2024-11-24T08:14:58,034 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting b697b04ad18b4baa9debb3b9e084a266, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1732436095738 2024-11-24T08:14:58,043 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:58,044 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/8dd20ec33cfd4d3e9d3d3de1d1a1eea8 is 50, key is test_row_0/A:col10/1732436095738/Put/seqid=0 2024-11-24T08:14:58,054 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#B#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:58,054 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/23f92aecee7f472ca062618038e2e1db is 50, key is test_row_0/B:col10/1732436095738/Put/seqid=0 2024-11-24T08:14:58,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742118_1294 (size=12847) 2024-11-24T08:14:58,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742119_1295 (size=12847) 2024-11-24T08:14:58,464 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/8dd20ec33cfd4d3e9d3d3de1d1a1eea8 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/8dd20ec33cfd4d3e9d3d3de1d1a1eea8 2024-11-24T08:14:58,469 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/A of 4e438c4f9c38da2a2238da06f655542f into 8dd20ec33cfd4d3e9d3d3de1d1a1eea8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:58,469 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:58,469 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/A, priority=13, startTime=1732436098031; duration=0sec 2024-11-24T08:14:58,469 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:14:58,469 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:A 2024-11-24T08:14:58,469 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:14:58,470 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:14:58,470 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/C is initiating minor compaction (all files) 2024-11-24T08:14:58,470 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/C in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:58,471 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/f6af2ae716fd4e6db73326800f690228, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/070fe0ea694e4d9cb1c08a5644d4dbe5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/e060bab8704e47ff8f38d3d62cd4e339] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.5 K 2024-11-24T08:14:58,471 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6af2ae716fd4e6db73326800f690228, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=541, earliestPutTs=1732436094512 2024-11-24T08:14:58,472 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 070fe0ea694e4d9cb1c08a5644d4dbe5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=577, earliestPutTs=1732436094605 2024-11-24T08:14:58,472 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e060bab8704e47ff8f38d3d62cd4e339, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1732436095738 2024-11-24T08:14:58,474 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/23f92aecee7f472ca062618038e2e1db as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/23f92aecee7f472ca062618038e2e1db 2024-11-24T08:14:58,479 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/B of 4e438c4f9c38da2a2238da06f655542f into 23f92aecee7f472ca062618038e2e1db(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:58,479 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:58,479 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/B, priority=13, startTime=1732436098031; duration=0sec 2024-11-24T08:14:58,479 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:58,479 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:B 2024-11-24T08:14:58,487 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#C#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:14:58,487 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/2ce92308fa6e422ca5697aadc4dfe2da is 50, key is test_row_0/C:col10/1732436095738/Put/seqid=0 2024-11-24T08:14:58,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742120_1296 (size=12847) 2024-11-24T08:14:58,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-24T08:14:58,530 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:14:58,530 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:14:58,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-24T08:14:58,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T08:14:58,532 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:14:58,533 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:14:58,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:14:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T08:14:58,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-24T08:14:58,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:58,686 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T08:14:58,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:14:58,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:58,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:14:58,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:58,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:14:58,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:14:58,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/05946b51b49d4daea1d3474c0120d8f1 is 50, key is test_row_0/A:col10/1732436096813/Put/seqid=0 2024-11-24T08:14:58,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742121_1297 (size=12301) 2024-11-24T08:14:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T08:14:58,897 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/2ce92308fa6e422ca5697aadc4dfe2da as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/2ce92308fa6e422ca5697aadc4dfe2da 2024-11-24T08:14:58,902 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/C of 4e438c4f9c38da2a2238da06f655542f into 2ce92308fa6e422ca5697aadc4dfe2da(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:14:58,902 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:58,902 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/C, priority=13, startTime=1732436098031; duration=0sec 2024-11-24T08:14:58,902 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:14:58,902 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:C 2024-11-24T08:14:58,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:14:58,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:14:58,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436158977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:58,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:58,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:58,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:58,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:58,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436158979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:58,982 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 4e438c4f9c38da2a2238da06f655542f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:14:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:58,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436158980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:58,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436158981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:59,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:59,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436159093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436159093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:59,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:59,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436159093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:59,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:59,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,095 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/05946b51b49d4daea1d3474c0120d8f1 2024-11-24T08:14:59,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:59,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,095 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:59,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/dd6c08031bda4204b627c5172ae252d5 is 50, key is test_row_0/B:col10/1732436096813/Put/seqid=0 2024-11-24T08:14:59,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742122_1298 (size=12301) 2024-11-24T08:14:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T08:14:59,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:59,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436159300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:59,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436159300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:59,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436159300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:59,509 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/dd6c08031bda4204b627c5172ae252d5 2024-11-24T08:14:59,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/68d1cf966b534c2199ebcf162b77d618 is 50, key is test_row_0/C:col10/1732436096813/Put/seqid=0 2024-11-24T08:14:59,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742123_1299 (size=12301) 2024-11-24T08:14:59,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:59,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436159610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:59,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436159610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436159610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:59,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T08:14:59,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:14:59,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436159919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:14:59,923 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in 4e438c4f9c38da2a2238da06f655542f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:14:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:14:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:14:59,963 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=614 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/68d1cf966b534c2199ebcf162b77d618 2024-11-24T08:14:59,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/05946b51b49d4daea1d3474c0120d8f1 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/05946b51b49d4daea1d3474c0120d8f1 2024-11-24T08:14:59,975 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/05946b51b49d4daea1d3474c0120d8f1, entries=150, sequenceid=614, filesize=12.0 K 2024-11-24T08:14:59,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/dd6c08031bda4204b627c5172ae252d5 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/dd6c08031bda4204b627c5172ae252d5 2024-11-24T08:14:59,991 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/dd6c08031bda4204b627c5172ae252d5, entries=150, sequenceid=614, filesize=12.0 K 2024-11-24T08:14:59,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/68d1cf966b534c2199ebcf162b77d618 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/68d1cf966b534c2199ebcf162b77d618 2024-11-24T08:14:59,997 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/68d1cf966b534c2199ebcf162b77d618, entries=150, sequenceid=614, filesize=12.0 K 2024-11-24T08:14:59,998 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4e438c4f9c38da2a2238da06f655542f in 1313ms, sequenceid=614, compaction requested=false 2024-11-24T08:14:59,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:14:59,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:14:59,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-24T08:14:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-24T08:15:00,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-24T08:15:00,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4670 sec 2024-11-24T08:15:00,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.4730 sec 2024-11-24T08:15:00,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:15:00,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T08:15:00,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:15:00,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:00,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:15:00,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:00,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:15:00,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:00,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/775d6e96896b4849b861fa90dc9f6354 is 50, key is test_row_0/A:col10/1732436098980/Put/seqid=0 2024-11-24T08:15:00,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742124_1300 (size=14741) 2024-11-24T08:15:00,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:00,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436160156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436160158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:00,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436160158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:00,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:00,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436160270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:00,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436160270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:00,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:00,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:00,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436160271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:00,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:00,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436160487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:00,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:00,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436160487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:00,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:00,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436160487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:00,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:00,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,492 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:00,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/775d6e96896b4849b861fa90dc9f6354 2024-11-24T08:15:00,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/3c54550f63ba46be993d07545f1a3c87 is 50, key is test_row_0/B:col10/1732436098980/Put/seqid=0 2024-11-24T08:15:00,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742125_1301 (size=12301) 2024-11-24T08:15:00,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/3c54550f63ba46be993d07545f1a3c87 2024-11-24T08:15:00,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/779bffe7a64a40e992bd8def3528042a is 50, key is test_row_0/C:col10/1732436098980/Put/seqid=0 2024-11-24T08:15:00,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742126_1302 (size=12301) 2024-11-24T08:15:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-24T08:15:00,681 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:00,682 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-24T08:15:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T08:15:00,684 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:00,685 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:00,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T08:15:00,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436160800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436160800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436160800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:00,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T08:15:00,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:00,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:15:00,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:00,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:00,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:00,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436160990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:00,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T08:15:00,993 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 4e438c4f9c38da2a2238da06f655542f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:00,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:00,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:15:00,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:00,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:00,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T08:15:01,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/779bffe7a64a40e992bd8def3528042a 2024-11-24T08:15:01,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/775d6e96896b4849b861fa90dc9f6354 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/775d6e96896b4849b861fa90dc9f6354 2024-11-24T08:15:01,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/775d6e96896b4849b861fa90dc9f6354, entries=200, sequenceid=629, filesize=14.4 K 2024-11-24T08:15:01,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/3c54550f63ba46be993d07545f1a3c87 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/3c54550f63ba46be993d07545f1a3c87 2024-11-24T08:15:01,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/3c54550f63ba46be993d07545f1a3c87, entries=150, sequenceid=629, filesize=12.0 K 2024-11-24T08:15:01,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/779bffe7a64a40e992bd8def3528042a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/779bffe7a64a40e992bd8def3528042a 2024-11-24T08:15:01,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/779bffe7a64a40e992bd8def3528042a, entries=150, sequenceid=629, filesize=12.0 K 2024-11-24T08:15:01,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 4e438c4f9c38da2a2238da06f655542f in 930ms, sequenceid=629, compaction requested=true 2024-11-24T08:15:01,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:01,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:01,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:01,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:01,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:01,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:01,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:01,053 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:01,053 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:01,054 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:01,054 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:01,054 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/B is initiating minor compaction (all files) 2024-11-24T08:15:01,054 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/A is initiating minor compaction (all files) 2024-11-24T08:15:01,054 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/B in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:01,054 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/A in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:01,055 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/23f92aecee7f472ca062618038e2e1db, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/dd6c08031bda4204b627c5172ae252d5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/3c54550f63ba46be993d07545f1a3c87] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.6 K 2024-11-24T08:15:01,055 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/8dd20ec33cfd4d3e9d3d3de1d1a1eea8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/05946b51b49d4daea1d3474c0120d8f1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/775d6e96896b4849b861fa90dc9f6354] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=39.0 K 2024-11-24T08:15:01,055 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dd20ec33cfd4d3e9d3d3de1d1a1eea8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1732436095738 2024-11-24T08:15:01,055 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 23f92aecee7f472ca062618038e2e1db, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1732436095738 2024-11-24T08:15:01,055 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting dd6c08031bda4204b627c5172ae252d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732436096807 2024-11-24T08:15:01,055 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05946b51b49d4daea1d3474c0120d8f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732436096807 2024-11-24T08:15:01,056 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 775d6e96896b4849b861fa90dc9f6354, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1732436098979 2024-11-24T08:15:01,056 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c54550f63ba46be993d07545f1a3c87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1732436098979 2024-11-24T08:15:01,066 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#B#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:01,067 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/cf5c1031fe5847ceb01ed4811578772d is 50, key is test_row_0/B:col10/1732436098980/Put/seqid=0 2024-11-24T08:15:01,068 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#A#compaction#255 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:01,068 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/2893fe00d9c4433881036248112d0723 is 50, key is test_row_0/A:col10/1732436098980/Put/seqid=0 2024-11-24T08:15:01,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742127_1303 (size=12949) 2024-11-24T08:15:01,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742128_1304 (size=12949) 2024-11-24T08:15:01,085 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/cf5c1031fe5847ceb01ed4811578772d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/cf5c1031fe5847ceb01ed4811578772d 2024-11-24T08:15:01,086 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/2893fe00d9c4433881036248112d0723 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/2893fe00d9c4433881036248112d0723 2024-11-24T08:15:01,095 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/B of 4e438c4f9c38da2a2238da06f655542f into cf5c1031fe5847ceb01ed4811578772d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:01,095 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:01,095 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/B, priority=13, startTime=1732436101053; duration=0sec 2024-11-24T08:15:01,095 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:01,095 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:B 2024-11-24T08:15:01,095 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:01,095 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/A of 4e438c4f9c38da2a2238da06f655542f into 2893fe00d9c4433881036248112d0723(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:01,095 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:01,096 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/A, priority=13, startTime=1732436101053; duration=0sec 2024-11-24T08:15:01,096 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:01,096 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:A 2024-11-24T08:15:01,097 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:01,097 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/C is initiating minor compaction (all files) 2024-11-24T08:15:01,097 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/C in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:01,097 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/2ce92308fa6e422ca5697aadc4dfe2da, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/68d1cf966b534c2199ebcf162b77d618, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/779bffe7a64a40e992bd8def3528042a] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.6 K 2024-11-24T08:15:01,098 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ce92308fa6e422ca5697aadc4dfe2da, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1732436095738 2024-11-24T08:15:01,099 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 68d1cf966b534c2199ebcf162b77d618, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=614, earliestPutTs=1732436096807 2024-11-24T08:15:01,100 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 779bffe7a64a40e992bd8def3528042a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1732436098979 2024-11-24T08:15:01,113 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#C#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:01,114 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/21e14c9d97c04d38be245a7b8a2b1134 is 50, key is test_row_0/C:col10/1732436098980/Put/seqid=0 2024-11-24T08:15:01,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742129_1305 (size=12949) 2024-11-24T08:15:01,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-24T08:15:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:01,148 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T08:15:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:15:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:15:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:15:01,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:01,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/86e2c09e2cff45e98bc53404dc4bc565 is 50, key is test_row_0/A:col10/1732436100154/Put/seqid=0 2024-11-24T08:15:01,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742130_1306 (size=12301) 2024-11-24T08:15:01,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T08:15:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:15:01,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:15:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:01,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436161329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:01,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:01,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436161331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436161331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436161439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436161439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:01,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436161439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:01,533 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/21e14c9d97c04d38be245a7b8a2b1134 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/21e14c9d97c04d38be245a7b8a2b1134 2024-11-24T08:15:01,538 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/C of 4e438c4f9c38da2a2238da06f655542f into 21e14c9d97c04d38be245a7b8a2b1134(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:01,538 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:01,538 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/C, priority=13, startTime=1732436101053; duration=0sec 2024-11-24T08:15:01,538 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:01,538 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:C 2024-11-24T08:15:01,558 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/86e2c09e2cff45e98bc53404dc4bc565 2024-11-24T08:15:01,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/f4d49c12d2e1451ca3d171857f7e481c is 50, key is test_row_0/B:col10/1732436100154/Put/seqid=0 2024-11-24T08:15:01,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742131_1307 (size=12301) 2024-11-24T08:15:01,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436161650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:01,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436161650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:01,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436161650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T08:15:01,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436161960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:01,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436161960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:01,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436161960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:01,971 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/f4d49c12d2e1451ca3d171857f7e481c 2024-11-24T08:15:01,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/bf03bf2091f142e987cb156b8136bd7f is 50, key is test_row_0/C:col10/1732436100154/Put/seqid=0 2024-11-24T08:15:01,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742132_1308 (size=12301) 2024-11-24T08:15:02,386 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=650 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/bf03bf2091f142e987cb156b8136bd7f 2024-11-24T08:15:02,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/86e2c09e2cff45e98bc53404dc4bc565 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/86e2c09e2cff45e98bc53404dc4bc565 2024-11-24T08:15:02,395 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/86e2c09e2cff45e98bc53404dc4bc565, entries=150, sequenceid=650, filesize=12.0 K 2024-11-24T08:15:02,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/f4d49c12d2e1451ca3d171857f7e481c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f4d49c12d2e1451ca3d171857f7e481c 2024-11-24T08:15:02,399 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f4d49c12d2e1451ca3d171857f7e481c, entries=150, sequenceid=650, filesize=12.0 K 2024-11-24T08:15:02,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/bf03bf2091f142e987cb156b8136bd7f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/bf03bf2091f142e987cb156b8136bd7f 2024-11-24T08:15:02,404 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/bf03bf2091f142e987cb156b8136bd7f, entries=150, sequenceid=650, filesize=12.0 K 2024-11-24T08:15:02,405 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 4e438c4f9c38da2a2238da06f655542f in 1257ms, sequenceid=650, compaction requested=false 2024-11-24T08:15:02,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:02,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:02,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-24T08:15:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-24T08:15:02,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-24T08:15:02,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7220 sec 2024-11-24T08:15:02,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.7300 sec 2024-11-24T08:15:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:15:02,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T08:15:02,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:15:02,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:02,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:15:02,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:02,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:15:02,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:02,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/fc06dbc8ce0d480396d2a65c02e6fef1 is 50, key is test_row_0/A:col10/1732436102473/Put/seqid=0 2024-11-24T08:15:02,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742133_1309 (size=17181) 2024-11-24T08:15:02,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/fc06dbc8ce0d480396d2a65c02e6fef1 2024-11-24T08:15:02,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/2783aadc02814ff399cc8f11780fd38e is 50, key is test_row_0/B:col10/1732436102473/Put/seqid=0 2024-11-24T08:15:02,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742134_1310 (size=12301) 2024-11-24T08:15:02,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:02,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436162502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:02,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:02,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:02,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436162503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:02,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:02,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:02,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436162508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:02,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:02,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436162609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:02,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:02,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436162620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:02,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436162620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:02,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:02,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436162819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:02,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:02,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436162829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:02,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436162830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:02,833 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-24T08:15:02,840 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:02,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-24T08:15:02,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T08:15:02,842 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:02,843 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:02,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:02,897 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/2783aadc02814ff399cc8f11780fd38e 2024-11-24T08:15:02,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/c329aeb80559444f9f9f898b766c84af is 50, key is test_row_0/C:col10/1732436102473/Put/seqid=0 2024-11-24T08:15:02,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742135_1311 (size=12301) 2024-11-24T08:15:02,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T08:15:02,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T08:15:02,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:02,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:15:02,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:02,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:02,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:02,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:03,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436163130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:03,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436163139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:03,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436163140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:03,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T08:15:03,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:03,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:15:03,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:03,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:03,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:03,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T08:15:03,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T08:15:03,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:03,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:15:03,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:03,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:03,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:03,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/c329aeb80559444f9f9f898b766c84af 2024-11-24T08:15:03,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/fc06dbc8ce0d480396d2a65c02e6fef1 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/fc06dbc8ce0d480396d2a65c02e6fef1 2024-11-24T08:15:03,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/fc06dbc8ce0d480396d2a65c02e6fef1, entries=250, sequenceid=666, filesize=16.8 K 2024-11-24T08:15:03,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/2783aadc02814ff399cc8f11780fd38e as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2783aadc02814ff399cc8f11780fd38e 2024-11-24T08:15:03,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2783aadc02814ff399cc8f11780fd38e, entries=150, sequenceid=666, filesize=12.0 K 2024-11-24T08:15:03,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/c329aeb80559444f9f9f898b766c84af as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/c329aeb80559444f9f9f898b766c84af 2024-11-24T08:15:03,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/c329aeb80559444f9f9f898b766c84af, entries=150, sequenceid=666, filesize=12.0 K 2024-11-24T08:15:03,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4e438c4f9c38da2a2238da06f655542f in 865ms, sequenceid=666, compaction requested=true 2024-11-24T08:15:03,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:03,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:03,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:03,339 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:03,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:03,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:03,339 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:03,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:03,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:03,340 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:03,340 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:03,340 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/A is initiating minor compaction (all files) 2024-11-24T08:15:03,340 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/B is initiating minor compaction (all files) 2024-11-24T08:15:03,340 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/B in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:03,340 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/A in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:03,340 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/2893fe00d9c4433881036248112d0723, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/86e2c09e2cff45e98bc53404dc4bc565, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/fc06dbc8ce0d480396d2a65c02e6fef1] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=41.4 K 2024-11-24T08:15:03,340 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/cf5c1031fe5847ceb01ed4811578772d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f4d49c12d2e1451ca3d171857f7e481c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2783aadc02814ff399cc8f11780fd38e] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.7 K 2024-11-24T08:15:03,341 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2893fe00d9c4433881036248112d0723, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1732436098979 2024-11-24T08:15:03,341 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting cf5c1031fe5847ceb01ed4811578772d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1732436098979 2024-11-24T08:15:03,341 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86e2c09e2cff45e98bc53404dc4bc565, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732436100154 2024-11-24T08:15:03,341 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d49c12d2e1451ca3d171857f7e481c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732436100154 2024-11-24T08:15:03,342 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc06dbc8ce0d480396d2a65c02e6fef1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1732436101322 2024-11-24T08:15:03,342 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2783aadc02814ff399cc8f11780fd38e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1732436101322 2024-11-24T08:15:03,359 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#A#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:03,359 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#B#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:03,360 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/d1f4c0b65e324e32ad13d5af4b28bc4d is 50, key is test_row_0/A:col10/1732436102473/Put/seqid=0 2024-11-24T08:15:03,360 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/06e4d9ebf5284fda86116e4d806f576c is 50, key is test_row_0/B:col10/1732436102473/Put/seqid=0 2024-11-24T08:15:03,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742136_1312 (size=13051) 2024-11-24T08:15:03,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742137_1313 (size=13051) 2024-11-24T08:15:03,374 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/d1f4c0b65e324e32ad13d5af4b28bc4d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/d1f4c0b65e324e32ad13d5af4b28bc4d 2024-11-24T08:15:03,378 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/A of 4e438c4f9c38da2a2238da06f655542f into d1f4c0b65e324e32ad13d5af4b28bc4d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:03,379 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:03,379 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/A, priority=13, startTime=1732436103339; duration=0sec 2024-11-24T08:15:03,379 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:03,379 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:A 2024-11-24T08:15:03,379 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:03,379 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:03,380 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/C is initiating minor compaction (all files) 2024-11-24T08:15:03,380 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/C in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:03,380 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/21e14c9d97c04d38be245a7b8a2b1134, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/bf03bf2091f142e987cb156b8136bd7f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/c329aeb80559444f9f9f898b766c84af] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.7 K 2024-11-24T08:15:03,380 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e14c9d97c04d38be245a7b8a2b1134, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=629, earliestPutTs=1732436098979 2024-11-24T08:15:03,380 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf03bf2091f142e987cb156b8136bd7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=650, earliestPutTs=1732436100154 2024-11-24T08:15:03,380 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c329aeb80559444f9f9f898b766c84af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1732436101322 2024-11-24T08:15:03,388 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#C#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:03,388 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/bcc8734daadb4dba88fce42d09cb323d is 50, key is test_row_0/C:col10/1732436102473/Put/seqid=0 2024-11-24T08:15:03,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742138_1314 (size=13051) 2024-11-24T08:15:03,409 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/bcc8734daadb4dba88fce42d09cb323d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/bcc8734daadb4dba88fce42d09cb323d 2024-11-24T08:15:03,416 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/C of 4e438c4f9c38da2a2238da06f655542f into bcc8734daadb4dba88fce42d09cb323d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:03,416 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:03,416 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/C, priority=13, startTime=1732436103339; duration=0sec 2024-11-24T08:15:03,416 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:03,416 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:C 2024-11-24T08:15:03,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-24T08:15:03,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:03,455 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T08:15:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:15:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:15:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:15:03,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:03,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/43e94aee590e4ec0bddc9f795f8532d4 is 50, key is test_row_0/A:col10/1732436102506/Put/seqid=0 2024-11-24T08:15:03,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742139_1315 (size=12301) 2024-11-24T08:15:03,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T08:15:03,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:15:03,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:15:03,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436163665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:03,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436163666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:03,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436163667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:03,770 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/06e4d9ebf5284fda86116e4d806f576c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/06e4d9ebf5284fda86116e4d806f576c 2024-11-24T08:15:03,775 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/B of 4e438c4f9c38da2a2238da06f655542f into 06e4d9ebf5284fda86116e4d806f576c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:03,775 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:03,775 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/B, priority=13, startTime=1732436103339; duration=0sec 2024-11-24T08:15:03,775 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:03,775 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:B 2024-11-24T08:15:03,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436163780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436163780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:03,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436163780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:03,867 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=685 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/43e94aee590e4ec0bddc9f795f8532d4 2024-11-24T08:15:03,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/94b0f427186e4765b32e75333dc0142e is 50, key is test_row_0/B:col10/1732436102506/Put/seqid=0 2024-11-24T08:15:03,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742140_1316 (size=12301) 2024-11-24T08:15:03,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55946 deadline: 1732436163970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:03,973 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in 4e438c4f9c38da2a2238da06f655542f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:03,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T08:15:03,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436163989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:03,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436163989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:03,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:03,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436163990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:04,280 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=685 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/94b0f427186e4765b32e75333dc0142e 2024-11-24T08:15:04,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/68d413929b0b4541aed20b05933d9919 is 50, key is test_row_0/C:col10/1732436102506/Put/seqid=0 2024-11-24T08:15:04,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742141_1317 (size=12301) 2024-11-24T08:15:04,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436164300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436164300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436164300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:04,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:04,693 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=685 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/68d413929b0b4541aed20b05933d9919 2024-11-24T08:15:04,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/43e94aee590e4ec0bddc9f795f8532d4 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/43e94aee590e4ec0bddc9f795f8532d4 2024-11-24T08:15:04,702 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/43e94aee590e4ec0bddc9f795f8532d4, entries=150, sequenceid=685, filesize=12.0 K 2024-11-24T08:15:04,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/94b0f427186e4765b32e75333dc0142e as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/94b0f427186e4765b32e75333dc0142e 2024-11-24T08:15:04,707 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/94b0f427186e4765b32e75333dc0142e, entries=150, sequenceid=685, filesize=12.0 K 2024-11-24T08:15:04,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/68d413929b0b4541aed20b05933d9919 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/68d413929b0b4541aed20b05933d9919 2024-11-24T08:15:04,714 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/68d413929b0b4541aed20b05933d9919, entries=150, sequenceid=685, filesize=12.0 K 2024-11-24T08:15:04,715 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4e438c4f9c38da2a2238da06f655542f in 1260ms, sequenceid=685, compaction requested=false 2024-11-24T08:15:04,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:04,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:04,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-24T08:15:04,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-24T08:15:04,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-24T08:15:04,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8740 sec 2024-11-24T08:15:04,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.8780 sec 2024-11-24T08:15:04,798 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T08:15:04,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:15:04,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T08:15:04,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:15:04,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:04,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:15:04,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:04,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:15:04,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:04,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/1c9a8abdfe394a148c396c1131eeced7 is 50, key is test_row_0/A:col10/1732436104812/Put/seqid=0 2024-11-24T08:15:04,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742142_1318 (size=12301) 2024-11-24T08:15:04,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:04,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:04,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436164837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:04,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436164837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:04,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:04,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436164839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:04,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:04,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436164949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:04,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436164949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,954 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:04,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436164950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:04,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:04,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-24T08:15:04,991 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:04,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:04,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-24T08:15:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T08:15:04,995 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:04,996 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:04,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:05,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:05,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55962 deadline: 1732436165040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:05,044 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 4e438c4f9c38da2a2238da06f655542f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T08:15:05,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-24T08:15:05,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:05,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:15:05,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:05,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:05,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436165160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:05,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436165160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:05,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436165160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:05,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=702 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/1c9a8abdfe394a148c396c1131eeced7 2024-11-24T08:15:05,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/66d19ce5cf3745d4997727413fdb9874 is 50, key is test_row_0/B:col10/1732436104812/Put/seqid=0 2024-11-24T08:15:05,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742143_1319 (size=12301) 2024-11-24T08:15:05,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=702 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/66d19ce5cf3745d4997727413fdb9874 2024-11-24T08:15:05,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/c3e2854d2e0046ceb4a4398a60d6c31b is 50, key is test_row_0/C:col10/1732436104812/Put/seqid=0 2024-11-24T08:15:05,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742144_1320 (size=12301) 2024-11-24T08:15:05,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=702 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/c3e2854d2e0046ceb4a4398a60d6c31b 2024-11-24T08:15:05,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/1c9a8abdfe394a148c396c1131eeced7 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/1c9a8abdfe394a148c396c1131eeced7 2024-11-24T08:15:05,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/1c9a8abdfe394a148c396c1131eeced7, entries=150, sequenceid=702, filesize=12.0 K 2024-11-24T08:15:05,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/66d19ce5cf3745d4997727413fdb9874 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/66d19ce5cf3745d4997727413fdb9874 2024-11-24T08:15:05,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/66d19ce5cf3745d4997727413fdb9874, entries=150, sequenceid=702, filesize=12.0 K 2024-11-24T08:15:05,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/c3e2854d2e0046ceb4a4398a60d6c31b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/c3e2854d2e0046ceb4a4398a60d6c31b 2024-11-24T08:15:05,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/c3e2854d2e0046ceb4a4398a60d6c31b, entries=150, sequenceid=702, filesize=12.0 K 2024-11-24T08:15:05,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4e438c4f9c38da2a2238da06f655542f in 483ms, sequenceid=702, compaction requested=true 2024-11-24T08:15:05,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:05,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:05,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:05,297 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:05,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:05,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:05,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:05,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:05,297 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:05,298 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:05,298 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/A is initiating minor compaction (all files) 2024-11-24T08:15:05,298 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/A in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:05,298 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/d1f4c0b65e324e32ad13d5af4b28bc4d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/43e94aee590e4ec0bddc9f795f8532d4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/1c9a8abdfe394a148c396c1131eeced7] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.8 K 2024-11-24T08:15:05,299 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1f4c0b65e324e32ad13d5af4b28bc4d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1732436101322 2024-11-24T08:15:05,300 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:05,300 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43e94aee590e4ec0bddc9f795f8532d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=685, earliestPutTs=1732436102492 2024-11-24T08:15:05,300 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/B is initiating minor compaction (all files) 2024-11-24T08:15:05,300 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/B in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:05,300 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/06e4d9ebf5284fda86116e4d806f576c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/94b0f427186e4765b32e75333dc0142e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/66d19ce5cf3745d4997727413fdb9874] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.8 K 2024-11-24T08:15:05,300 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c9a8abdfe394a148c396c1131eeced7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=702, earliestPutTs=1732436103664 2024-11-24T08:15:05,300 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 06e4d9ebf5284fda86116e4d806f576c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1732436101322 2024-11-24T08:15:05,301 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 94b0f427186e4765b32e75333dc0142e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=685, earliestPutTs=1732436102492 2024-11-24T08:15:05,301 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 66d19ce5cf3745d4997727413fdb9874, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=702, earliestPutTs=1732436103664 2024-11-24T08:15:05,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-24T08:15:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:05,302 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T08:15:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:15:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:15:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:15:05,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T08:15:05,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/3a6c676aa5784c6a9a78425d3e7da5c0 is 50, key is test_row_0/A:col10/1732436104835/Put/seqid=0 2024-11-24T08:15:05,320 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#A#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:05,321 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/452ab75f304044c2869fd2d3e04f5d00 is 50, key is test_row_0/A:col10/1732436104812/Put/seqid=0 2024-11-24T08:15:05,347 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#B#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:05,347 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/f2b11c8861cf455887a835b7ff05f442 is 50, key is test_row_0/B:col10/1732436104812/Put/seqid=0 2024-11-24T08:15:05,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742146_1322 (size=12301) 2024-11-24T08:15:05,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742145_1321 (size=13153) 2024-11-24T08:15:05,359 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=718 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/3a6c676aa5784c6a9a78425d3e7da5c0 2024-11-24T08:15:05,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742147_1323 (size=13153) 2024-11-24T08:15:05,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/b7f4742ca46242648313d3446f85547a is 50, key is test_row_0/B:col10/1732436104835/Put/seqid=0 2024-11-24T08:15:05,379 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/f2b11c8861cf455887a835b7ff05f442 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f2b11c8861cf455887a835b7ff05f442 2024-11-24T08:15:05,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742148_1324 (size=12301) 2024-11-24T08:15:05,384 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/B of 4e438c4f9c38da2a2238da06f655542f into f2b11c8861cf455887a835b7ff05f442(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:05,384 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:05,384 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/B, priority=13, startTime=1732436105297; duration=0sec 2024-11-24T08:15:05,385 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:05,385 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:B 2024-11-24T08:15:05,385 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:05,385 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=718 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/b7f4742ca46242648313d3446f85547a 2024-11-24T08:15:05,386 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:05,386 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/C is initiating minor compaction (all files) 2024-11-24T08:15:05,386 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/C in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:05,387 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/bcc8734daadb4dba88fce42d09cb323d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/68d413929b0b4541aed20b05933d9919, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/c3e2854d2e0046ceb4a4398a60d6c31b] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.8 K 2024-11-24T08:15:05,387 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting bcc8734daadb4dba88fce42d09cb323d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1732436101322 2024-11-24T08:15:05,387 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 68d413929b0b4541aed20b05933d9919, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=685, earliestPutTs=1732436102492 2024-11-24T08:15:05,388 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting c3e2854d2e0046ceb4a4398a60d6c31b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=702, earliestPutTs=1732436103664 2024-11-24T08:15:05,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/36c01684f9124a2bacf961fc386077ff is 50, key is test_row_0/C:col10/1732436104835/Put/seqid=0 2024-11-24T08:15:05,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742149_1325 (size=12301) 2024-11-24T08:15:05,403 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#C#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:05,404 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/1dc9b6264f9a480ca75aa01fa20e539f is 50, key is test_row_0/C:col10/1732436104812/Put/seqid=0 2024-11-24T08:15:05,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742150_1326 (size=13153) 2024-11-24T08:15:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:15:05,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. as already flushing 2024-11-24T08:15:05,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436165492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:05,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436165494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:05,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436165496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:05,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436165599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:05,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436165610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:05,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:05,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436165610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:05,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:05,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T08:15:05,764 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/452ab75f304044c2869fd2d3e04f5d00 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/452ab75f304044c2869fd2d3e04f5d00 2024-11-24T08:15:05,769 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/A of 4e438c4f9c38da2a2238da06f655542f into 452ab75f304044c2869fd2d3e04f5d00(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:05,769 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:05,769 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/A, priority=13, startTime=1732436105296; duration=0sec 2024-11-24T08:15:05,769 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:05,769 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:A 2024-11-24T08:15:05,803 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=718 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/36c01684f9124a2bacf961fc386077ff 2024-11-24T08:15:05,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/3a6c676aa5784c6a9a78425d3e7da5c0 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/3a6c676aa5784c6a9a78425d3e7da5c0 2024-11-24T08:15:05,812 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/3a6c676aa5784c6a9a78425d3e7da5c0, entries=150, sequenceid=718, filesize=12.0 K 2024-11-24T08:15:05,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:05,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/b7f4742ca46242648313d3446f85547a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/b7f4742ca46242648313d3446f85547a 2024-11-24T08:15:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55896 deadline: 1732436165810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:05,817 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/1dc9b6264f9a480ca75aa01fa20e539f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/1dc9b6264f9a480ca75aa01fa20e539f 2024-11-24T08:15:05,818 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/b7f4742ca46242648313d3446f85547a, entries=150, sequenceid=718, filesize=12.0 K 2024-11-24T08:15:05,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/36c01684f9124a2bacf961fc386077ff as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/36c01684f9124a2bacf961fc386077ff 2024-11-24T08:15:05,822 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/C of 4e438c4f9c38da2a2238da06f655542f into 1dc9b6264f9a480ca75aa01fa20e539f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:05,822 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:05,822 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/C, priority=13, startTime=1732436105297; duration=0sec 2024-11-24T08:15:05,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:05,822 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55950 deadline: 1732436165820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:05,822 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:C 2024-11-24T08:15:05,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:05,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55954 deadline: 1732436165820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:05,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:05,823 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/36c01684f9124a2bacf961fc386077ff, entries=150, sequenceid=718, filesize=12.0 K 2024-11-24T08:15:05,824 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 4e438c4f9c38da2a2238da06f655542f in 522ms, sequenceid=718, compaction requested=false 2024-11-24T08:15:05,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:05,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:05,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-24T08:15:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:05,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-24T08:15:05,828 INFO [Thread-1153 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1153. 2024-11-24T08:15:05,829 DEBUG [Thread-1153 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:05,829 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:05,829 INFO [Thread-1155 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1155. 2024-11-24T08:15:05,829 DEBUG [Thread-1155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:05,829 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:05,829 DEBUG [Thread-1153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:05,829 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:05,829 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:05,829 DEBUG [Thread-1155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:05,831 INFO [Thread-1149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1149. 2024-11-24T08:15:05,831 DEBUG [Thread-1149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:05,831 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:05,831 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:05,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:05,832 INFO [Thread-1157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1157. 2024-11-24T08:15:05,832 DEBUG [Thread-1157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:05,832 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:05,832 INFO [Thread-1151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1151. 2024-11-24T08:15:05,832 DEBUG [Thread-1157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:05,832 DEBUG [Thread-1151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:05,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:05,832 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:05,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:05,833 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:05,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-24T08:15:05,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 833 msec 2024-11-24T08:15:05,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 841 msec 2024-11-24T08:15:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:15:06,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T08:15:06,121 INFO [Thread-1146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1146. 2024-11-24T08:15:06,121 DEBUG [Thread-1146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:06,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:15:06,121 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:06,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:06,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:15:06,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:06,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:15:06,121 DEBUG [Thread-1146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:06,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:06,121 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:06,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/06e4a6851f504157bc58848b18a9f918 is 50, key is test_row_0/A:col10/1732436105484/Put/seqid=0 2024-11-24T08:15:06,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742151_1327 (size=12301) 2024-11-24T08:15:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-24T08:15:06,130 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:06,131 INFO [Thread-1138 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1138. 2024-11-24T08:15:06,131 DEBUG [Thread-1138 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:06,131 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:06,131 DEBUG [Thread-1138 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:06,131 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:06,141 INFO [Thread-1142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1142. 2024-11-24T08:15:06,141 DEBUG [Thread-1142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:06,141 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:06,141 DEBUG [Thread-1142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:06,141 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:06,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=738 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/06e4a6851f504157bc58848b18a9f918 2024-11-24T08:15:06,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/51c8408a5dfe4feea326c9ffb6223ceb is 50, key is test_row_0/B:col10/1732436105484/Put/seqid=0 2024-11-24T08:15:06,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742152_1328 (size=12301) 2024-11-24T08:15:06,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=738 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/51c8408a5dfe4feea326c9ffb6223ceb 2024-11-24T08:15:06,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/0f623e55598e41d385315f6ede328a3f is 50, key is test_row_0/C:col10/1732436105484/Put/seqid=0 2024-11-24T08:15:06,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742153_1329 (size=12301) 2024-11-24T08:15:07,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=738 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/0f623e55598e41d385315f6ede328a3f 2024-11-24T08:15:07,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/06e4a6851f504157bc58848b18a9f918 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/06e4a6851f504157bc58848b18a9f918 2024-11-24T08:15:07,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/06e4a6851f504157bc58848b18a9f918, entries=150, sequenceid=738, filesize=12.0 K 2024-11-24T08:15:07,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/51c8408a5dfe4feea326c9ffb6223ceb as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/51c8408a5dfe4feea326c9ffb6223ceb 2024-11-24T08:15:07,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/51c8408a5dfe4feea326c9ffb6223ceb, entries=150, sequenceid=738, filesize=12.0 K 2024-11-24T08:15:07,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/0f623e55598e41d385315f6ede328a3f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/0f623e55598e41d385315f6ede328a3f 2024-11-24T08:15:07,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/0f623e55598e41d385315f6ede328a3f, entries=150, sequenceid=738, filesize=12.0 K 2024-11-24T08:15:07,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=13.42 KB/13740 for 4e438c4f9c38da2a2238da06f655542f in 1244ms, sequenceid=738, compaction requested=true 2024-11-24T08:15:07,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:07,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:07,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:07,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:07,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:07,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4e438c4f9c38da2a2238da06f655542f:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:07,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:07,365 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:07,365 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:07,366 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:07,366 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:07,366 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/A is initiating minor compaction (all files) 2024-11-24T08:15:07,366 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/A in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:07,366 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/B is initiating minor compaction (all files) 2024-11-24T08:15:07,366 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/452ab75f304044c2869fd2d3e04f5d00, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/3a6c676aa5784c6a9a78425d3e7da5c0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/06e4a6851f504157bc58848b18a9f918] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.9 K 2024-11-24T08:15:07,366 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/B in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:07,366 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f2b11c8861cf455887a835b7ff05f442, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/b7f4742ca46242648313d3446f85547a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/51c8408a5dfe4feea326c9ffb6223ceb] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.9 K 2024-11-24T08:15:07,367 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 452ab75f304044c2869fd2d3e04f5d00, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=702, earliestPutTs=1732436103664 2024-11-24T08:15:07,367 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting f2b11c8861cf455887a835b7ff05f442, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=702, earliestPutTs=1732436103664 2024-11-24T08:15:07,367 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a6c676aa5784c6a9a78425d3e7da5c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=718, earliestPutTs=1732436104827 2024-11-24T08:15:07,367 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting b7f4742ca46242648313d3446f85547a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=718, earliestPutTs=1732436104827 2024-11-24T08:15:07,367 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06e4a6851f504157bc58848b18a9f918, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=738, earliestPutTs=1732436105484 2024-11-24T08:15:07,367 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 51c8408a5dfe4feea326c9ffb6223ceb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=738, earliestPutTs=1732436105484 2024-11-24T08:15:07,373 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#A#compaction#282 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:07,374 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#B#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:07,374 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/bc1769bed8a34fc09057feafe77ecf6b is 50, key is test_row_0/A:col10/1732436105484/Put/seqid=0 2024-11-24T08:15:07,374 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/de5ad085f39f41d8a28bb326641dcdef is 50, key is test_row_0/B:col10/1732436105484/Put/seqid=0 2024-11-24T08:15:07,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742154_1330 (size=13255) 2024-11-24T08:15:07,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742155_1331 (size=13255) 2024-11-24T08:15:07,782 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/bc1769bed8a34fc09057feafe77ecf6b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/bc1769bed8a34fc09057feafe77ecf6b 2024-11-24T08:15:07,784 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/de5ad085f39f41d8a28bb326641dcdef as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/de5ad085f39f41d8a28bb326641dcdef 2024-11-24T08:15:07,786 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/A of 4e438c4f9c38da2a2238da06f655542f into bc1769bed8a34fc09057feafe77ecf6b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:07,786 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:07,786 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/A, priority=13, startTime=1732436107365; duration=0sec 2024-11-24T08:15:07,786 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:07,786 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:A 2024-11-24T08:15:07,786 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:07,787 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:07,787 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 4e438c4f9c38da2a2238da06f655542f/C is initiating minor compaction (all files) 2024-11-24T08:15:07,787 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4e438c4f9c38da2a2238da06f655542f/C in TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:07,788 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/1dc9b6264f9a480ca75aa01fa20e539f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/36c01684f9124a2bacf961fc386077ff, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/0f623e55598e41d385315f6ede328a3f] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp, totalSize=36.9 K 2024-11-24T08:15:07,788 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/B of 4e438c4f9c38da2a2238da06f655542f into de5ad085f39f41d8a28bb326641dcdef(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:07,788 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:07,788 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/B, priority=13, startTime=1732436107365; duration=0sec 2024-11-24T08:15:07,788 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dc9b6264f9a480ca75aa01fa20e539f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=702, earliestPutTs=1732436103664 2024-11-24T08:15:07,788 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:07,788 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:B 2024-11-24T08:15:07,788 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36c01684f9124a2bacf961fc386077ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=718, earliestPutTs=1732436104827 2024-11-24T08:15:07,788 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f623e55598e41d385315f6ede328a3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=738, earliestPutTs=1732436105484 2024-11-24T08:15:07,793 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4e438c4f9c38da2a2238da06f655542f#C#compaction#284 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:07,794 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/22b390fb3036423cbad325649d45fd99 is 50, key is test_row_0/C:col10/1732436105484/Put/seqid=0 2024-11-24T08:15:07,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742156_1332 (size=13255) 2024-11-24T08:15:08,202 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/22b390fb3036423cbad325649d45fd99 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/22b390fb3036423cbad325649d45fd99 2024-11-24T08:15:08,206 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4e438c4f9c38da2a2238da06f655542f/C of 4e438c4f9c38da2a2238da06f655542f into 22b390fb3036423cbad325649d45fd99(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:08,206 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4e438c4f9c38da2a2238da06f655542f: 2024-11-24T08:15:08,206 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., storeName=4e438c4f9c38da2a2238da06f655542f/C, priority=13, startTime=1732436107365; duration=0sec 2024-11-24T08:15:08,206 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:08,206 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4e438c4f9c38da2a2238da06f655542f:C 2024-11-24T08:15:14,081 INFO [Thread-1140 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1140. 2024-11-24T08:15:14,081 DEBUG [Thread-1140 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:14,081 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:14,082 DEBUG [Thread-1140 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:14,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:15,111 INFO [Thread-1144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1144. 2024-11-24T08:15:15,111 DEBUG [Thread-1144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:15,111 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:15,111 DEBUG [Thread-1144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:15,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:15,111 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T08:15:15,112 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 147 2024-11-24T08:15:15,112 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 119 2024-11-24T08:15:15,112 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-24T08:15:15,112 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 162 2024-11-24T08:15:15,112 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-24T08:15:15,112 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T08:15:15,112 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3050 2024-11-24T08:15:15,112 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3256 2024-11-24T08:15:15,112 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3113 2024-11-24T08:15:15,112 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3053 2024-11-24T08:15:15,112 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3261 2024-11-24T08:15:15,112 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T08:15:15,112 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T08:15:15,112 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:15:15,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:15,112 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:15,112 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:15,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T08:15:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T08:15:15,115 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436115115"}]},"ts":"1732436115115"} 2024-11-24T08:15:15,117 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T08:15:15,117 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T08:15:15,117 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T08:15:15,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e438c4f9c38da2a2238da06f655542f, UNASSIGN}] 2024-11-24T08:15:15,119 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e438c4f9c38da2a2238da06f655542f, UNASSIGN 2024-11-24T08:15:15,120 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=4e438c4f9c38da2a2238da06f655542f, regionState=CLOSING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:15:15,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e438c4f9c38da2a2238da06f655542f, UNASSIGN because future has completed 2024-11-24T08:15:15,121 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T08:15:15,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:15:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T08:15:15,273 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:15:15,273 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T08:15:15,273 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 4e438c4f9c38da2a2238da06f655542f, disabling compactions & flushes 2024-11-24T08:15:15,273 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:15,273 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:15,273 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. after waiting 0 ms 2024-11-24T08:15:15,273 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:15,273 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing 4e438c4f9c38da2a2238da06f655542f 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-24T08:15:15,274 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=A 2024-11-24T08:15:15,274 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:15,274 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=B 2024-11-24T08:15:15,274 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:15,274 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4e438c4f9c38da2a2238da06f655542f, store=C 2024-11-24T08:15:15,274 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:15,277 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/62245d7797da49c4b15bbd6f0c4d290d is 50, key is test_row_1/A:col10/1732436114080/Put/seqid=0 2024-11-24T08:15:15,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742157_1333 (size=9857) 2024-11-24T08:15:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T08:15:15,681 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/62245d7797da49c4b15bbd6f0c4d290d 2024-11-24T08:15:15,688 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/7063859cae2c4d19a1424dd9b5d76fcf is 50, key is test_row_1/B:col10/1732436114080/Put/seqid=0 2024-11-24T08:15:15,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742158_1334 (size=9857) 2024-11-24T08:15:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T08:15:16,092 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/7063859cae2c4d19a1424dd9b5d76fcf 2024-11-24T08:15:16,099 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/d05556f682864f188abb9ebf7baa46a7 is 50, key is test_row_1/C:col10/1732436114080/Put/seqid=0 2024-11-24T08:15:16,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742159_1335 (size=9857) 2024-11-24T08:15:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T08:15:16,505 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/d05556f682864f188abb9ebf7baa46a7 2024-11-24T08:15:16,509 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/A/62245d7797da49c4b15bbd6f0c4d290d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/62245d7797da49c4b15bbd6f0c4d290d 2024-11-24T08:15:16,512 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/62245d7797da49c4b15bbd6f0c4d290d, entries=100, sequenceid=748, filesize=9.6 K 2024-11-24T08:15:16,513 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/B/7063859cae2c4d19a1424dd9b5d76fcf as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/7063859cae2c4d19a1424dd9b5d76fcf 2024-11-24T08:15:16,516 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/7063859cae2c4d19a1424dd9b5d76fcf, entries=100, sequenceid=748, filesize=9.6 K 2024-11-24T08:15:16,516 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/.tmp/C/d05556f682864f188abb9ebf7baa46a7 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/d05556f682864f188abb9ebf7baa46a7 2024-11-24T08:15:16,520 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/d05556f682864f188abb9ebf7baa46a7, entries=100, sequenceid=748, filesize=9.6 K 2024-11-24T08:15:16,520 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4e438c4f9c38da2a2238da06f655542f in 1247ms, sequenceid=748, compaction requested=false 2024-11-24T08:15:16,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/1ac59d7499be43a7aabe67cfddc8bd62, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/20e80ec208fa4d0b81043eca3667c6c7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/0ae94faafd9f41c688d0004b7c7f576c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/f1160281797a4a1cb34401285f5ad640, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/71a854f417c0439b8f92a74747b2cb50, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/b7aa96316d334e44a118d5b88851ad52, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/52d62f4b89b847338cf10d0c93ad5ffb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/ffc5bababae246348d863ab5a0dd6b55, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/6f77986c5a2849e19283d470e2f77544, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/798d1fd5bcee426cbc5ea0bc5c572dc2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/ad9e7b19e27d4afcae0ae99b3dec8de6, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/cb204720cfe246dd961354905169d99e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/299edc49114646f39d7cce66d231817e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/00b38a4025964b51a099a6e920414850, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/395cb25699bd4e2eb9db512538db64f2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/8dd20ec33cfd4d3e9d3d3de1d1a1eea8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/e59f006146cf4c81a3ab2795ab2b3074, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/05946b51b49d4daea1d3474c0120d8f1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/775d6e96896b4849b861fa90dc9f6354, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/2893fe00d9c4433881036248112d0723, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/86e2c09e2cff45e98bc53404dc4bc565, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/fc06dbc8ce0d480396d2a65c02e6fef1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/d1f4c0b65e324e32ad13d5af4b28bc4d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/43e94aee590e4ec0bddc9f795f8532d4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/452ab75f304044c2869fd2d3e04f5d00, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/1c9a8abdfe394a148c396c1131eeced7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/3a6c676aa5784c6a9a78425d3e7da5c0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/06e4a6851f504157bc58848b18a9f918] to archive 2024-11-24T08:15:16,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:15:16,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/1ac59d7499be43a7aabe67cfddc8bd62 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/1ac59d7499be43a7aabe67cfddc8bd62 2024-11-24T08:15:16,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/20e80ec208fa4d0b81043eca3667c6c7 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/20e80ec208fa4d0b81043eca3667c6c7 2024-11-24T08:15:16,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/0ae94faafd9f41c688d0004b7c7f576c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/0ae94faafd9f41c688d0004b7c7f576c 2024-11-24T08:15:16,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/f1160281797a4a1cb34401285f5ad640 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/f1160281797a4a1cb34401285f5ad640 2024-11-24T08:15:16,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/71a854f417c0439b8f92a74747b2cb50 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/71a854f417c0439b8f92a74747b2cb50 2024-11-24T08:15:16,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/b7aa96316d334e44a118d5b88851ad52 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/b7aa96316d334e44a118d5b88851ad52 2024-11-24T08:15:16,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/52d62f4b89b847338cf10d0c93ad5ffb to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/52d62f4b89b847338cf10d0c93ad5ffb 2024-11-24T08:15:16,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/ffc5bababae246348d863ab5a0dd6b55 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/ffc5bababae246348d863ab5a0dd6b55 2024-11-24T08:15:16,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/6f77986c5a2849e19283d470e2f77544 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/6f77986c5a2849e19283d470e2f77544 2024-11-24T08:15:16,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/798d1fd5bcee426cbc5ea0bc5c572dc2 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/798d1fd5bcee426cbc5ea0bc5c572dc2 2024-11-24T08:15:16,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/ad9e7b19e27d4afcae0ae99b3dec8de6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/ad9e7b19e27d4afcae0ae99b3dec8de6 2024-11-24T08:15:16,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/cb204720cfe246dd961354905169d99e to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/cb204720cfe246dd961354905169d99e 2024-11-24T08:15:16,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/299edc49114646f39d7cce66d231817e to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/299edc49114646f39d7cce66d231817e 2024-11-24T08:15:16,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/00b38a4025964b51a099a6e920414850 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/00b38a4025964b51a099a6e920414850 2024-11-24T08:15:16,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/395cb25699bd4e2eb9db512538db64f2 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/395cb25699bd4e2eb9db512538db64f2 2024-11-24T08:15:16,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/8dd20ec33cfd4d3e9d3d3de1d1a1eea8 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/8dd20ec33cfd4d3e9d3d3de1d1a1eea8 2024-11-24T08:15:16,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/e59f006146cf4c81a3ab2795ab2b3074 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/e59f006146cf4c81a3ab2795ab2b3074 2024-11-24T08:15:16,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/05946b51b49d4daea1d3474c0120d8f1 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/05946b51b49d4daea1d3474c0120d8f1 2024-11-24T08:15:16,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/775d6e96896b4849b861fa90dc9f6354 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/775d6e96896b4849b861fa90dc9f6354 2024-11-24T08:15:16,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/2893fe00d9c4433881036248112d0723 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/2893fe00d9c4433881036248112d0723 2024-11-24T08:15:16,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/86e2c09e2cff45e98bc53404dc4bc565 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/86e2c09e2cff45e98bc53404dc4bc565 2024-11-24T08:15:16,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/fc06dbc8ce0d480396d2a65c02e6fef1 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/fc06dbc8ce0d480396d2a65c02e6fef1 2024-11-24T08:15:16,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/d1f4c0b65e324e32ad13d5af4b28bc4d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/d1f4c0b65e324e32ad13d5af4b28bc4d 2024-11-24T08:15:16,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/43e94aee590e4ec0bddc9f795f8532d4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/43e94aee590e4ec0bddc9f795f8532d4 2024-11-24T08:15:16,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/452ab75f304044c2869fd2d3e04f5d00 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/452ab75f304044c2869fd2d3e04f5d00 2024-11-24T08:15:16,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/1c9a8abdfe394a148c396c1131eeced7 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/1c9a8abdfe394a148c396c1131eeced7 2024-11-24T08:15:16,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/3a6c676aa5784c6a9a78425d3e7da5c0 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/3a6c676aa5784c6a9a78425d3e7da5c0 2024-11-24T08:15:16,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/06e4a6851f504157bc58848b18a9f918 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/06e4a6851f504157bc58848b18a9f918 2024-11-24T08:15:16,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/262854abf02e445bbe5285618eb6cc84, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/3a631affeeee4d26b870675a67c7d549, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/65027a2805b244a895645c70abfb428d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/775293459e53446197f02d303cd04a3a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2eda52ad1a77466a8632425d0f6165ea, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/dff6ae1021ed41cebcc80cc2245729fd, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f19ae6d4557946c7bf52f923b41346ec, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/958b2372ba8b44d1bd434f344d5870ec, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2243a6645d4642188cbfbb735444ec8c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/21adf159b9434e95ace50d988a011cfa, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/e4df57be80ca411b88e49844fcf21c68, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/e60b8abcb3cb4e59b0c80cb1c57b5303, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/60a640c14ff3454a90f37064ad5ee9ef, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/99f01cf9f17744abb3d4b4ff50d24c9c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/fd910f38e44741ed9890a83142ae0ba0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/23f92aecee7f472ca062618038e2e1db, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/b697b04ad18b4baa9debb3b9e084a266, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/dd6c08031bda4204b627c5172ae252d5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/cf5c1031fe5847ceb01ed4811578772d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/3c54550f63ba46be993d07545f1a3c87, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f4d49c12d2e1451ca3d171857f7e481c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/06e4d9ebf5284fda86116e4d806f576c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2783aadc02814ff399cc8f11780fd38e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/94b0f427186e4765b32e75333dc0142e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f2b11c8861cf455887a835b7ff05f442, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/66d19ce5cf3745d4997727413fdb9874, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/b7f4742ca46242648313d3446f85547a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/51c8408a5dfe4feea326c9ffb6223ceb] to archive 2024-11-24T08:15:16,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:15:16,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/262854abf02e445bbe5285618eb6cc84 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/262854abf02e445bbe5285618eb6cc84 2024-11-24T08:15:16,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/3a631affeeee4d26b870675a67c7d549 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/3a631affeeee4d26b870675a67c7d549 2024-11-24T08:15:16,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/65027a2805b244a895645c70abfb428d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/65027a2805b244a895645c70abfb428d 2024-11-24T08:15:16,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/775293459e53446197f02d303cd04a3a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/775293459e53446197f02d303cd04a3a 2024-11-24T08:15:16,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2eda52ad1a77466a8632425d0f6165ea to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2eda52ad1a77466a8632425d0f6165ea 2024-11-24T08:15:16,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/dff6ae1021ed41cebcc80cc2245729fd to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/dff6ae1021ed41cebcc80cc2245729fd 2024-11-24T08:15:16,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f19ae6d4557946c7bf52f923b41346ec to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f19ae6d4557946c7bf52f923b41346ec 2024-11-24T08:15:16,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/958b2372ba8b44d1bd434f344d5870ec to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/958b2372ba8b44d1bd434f344d5870ec 2024-11-24T08:15:16,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2243a6645d4642188cbfbb735444ec8c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2243a6645d4642188cbfbb735444ec8c 2024-11-24T08:15:16,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/21adf159b9434e95ace50d988a011cfa to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/21adf159b9434e95ace50d988a011cfa 2024-11-24T08:15:16,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/e4df57be80ca411b88e49844fcf21c68 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/e4df57be80ca411b88e49844fcf21c68 2024-11-24T08:15:16,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/e60b8abcb3cb4e59b0c80cb1c57b5303 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/e60b8abcb3cb4e59b0c80cb1c57b5303 2024-11-24T08:15:16,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/60a640c14ff3454a90f37064ad5ee9ef to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/60a640c14ff3454a90f37064ad5ee9ef 2024-11-24T08:15:16,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/99f01cf9f17744abb3d4b4ff50d24c9c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/99f01cf9f17744abb3d4b4ff50d24c9c 2024-11-24T08:15:16,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/fd910f38e44741ed9890a83142ae0ba0 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/fd910f38e44741ed9890a83142ae0ba0 2024-11-24T08:15:16,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/23f92aecee7f472ca062618038e2e1db to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/23f92aecee7f472ca062618038e2e1db 2024-11-24T08:15:16,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/b697b04ad18b4baa9debb3b9e084a266 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/b697b04ad18b4baa9debb3b9e084a266 2024-11-24T08:15:16,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/dd6c08031bda4204b627c5172ae252d5 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/dd6c08031bda4204b627c5172ae252d5 2024-11-24T08:15:16,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/cf5c1031fe5847ceb01ed4811578772d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/cf5c1031fe5847ceb01ed4811578772d 2024-11-24T08:15:16,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/3c54550f63ba46be993d07545f1a3c87 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/3c54550f63ba46be993d07545f1a3c87 2024-11-24T08:15:16,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f4d49c12d2e1451ca3d171857f7e481c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f4d49c12d2e1451ca3d171857f7e481c 2024-11-24T08:15:16,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/06e4d9ebf5284fda86116e4d806f576c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/06e4d9ebf5284fda86116e4d806f576c 2024-11-24T08:15:16,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2783aadc02814ff399cc8f11780fd38e to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/2783aadc02814ff399cc8f11780fd38e 2024-11-24T08:15:16,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/94b0f427186e4765b32e75333dc0142e to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/94b0f427186e4765b32e75333dc0142e 2024-11-24T08:15:16,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f2b11c8861cf455887a835b7ff05f442 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/f2b11c8861cf455887a835b7ff05f442 2024-11-24T08:15:16,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/66d19ce5cf3745d4997727413fdb9874 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/66d19ce5cf3745d4997727413fdb9874 2024-11-24T08:15:16,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/b7f4742ca46242648313d3446f85547a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/b7f4742ca46242648313d3446f85547a 2024-11-24T08:15:16,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/51c8408a5dfe4feea326c9ffb6223ceb to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/51c8408a5dfe4feea326c9ffb6223ceb 2024-11-24T08:15:16,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/e3f40f2379a146a581f8ebfc010e5b0f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/1886bdccc27c48cea91402188ea0a19f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/35c82d91fdeb446c83c09772a7e6aa18, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/6c2bc7eadcb74e52ae34e183e3bc6840, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/705de8ccf1bb45619101c77ff260fd50, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/557052710fe24908bf05359a1a708f6d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/b5318c7600054d449b51603b5b6d88e3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/b9c3f49d70b041c6901a34fb84d6b9cb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/a94240476afe403ca6ec3134434fbab8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/16b96be86774407e95f0acd91705c0e1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/9898ee77872b4642b80f3534d949dd31, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/de88a25d875845e8bd7206b5a7989271, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/f6af2ae716fd4e6db73326800f690228, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/7c26f67b25ce4621b2cc67c6bc8f00e4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/070fe0ea694e4d9cb1c08a5644d4dbe5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/2ce92308fa6e422ca5697aadc4dfe2da, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/e060bab8704e47ff8f38d3d62cd4e339, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/68d1cf966b534c2199ebcf162b77d618, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/21e14c9d97c04d38be245a7b8a2b1134, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/779bffe7a64a40e992bd8def3528042a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/bf03bf2091f142e987cb156b8136bd7f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/bcc8734daadb4dba88fce42d09cb323d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/c329aeb80559444f9f9f898b766c84af, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/68d413929b0b4541aed20b05933d9919, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/1dc9b6264f9a480ca75aa01fa20e539f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/c3e2854d2e0046ceb4a4398a60d6c31b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/36c01684f9124a2bacf961fc386077ff, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/0f623e55598e41d385315f6ede328a3f] to archive 2024-11-24T08:15:16,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:15:16,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/e3f40f2379a146a581f8ebfc010e5b0f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/e3f40f2379a146a581f8ebfc010e5b0f 2024-11-24T08:15:16,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/1886bdccc27c48cea91402188ea0a19f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/1886bdccc27c48cea91402188ea0a19f 2024-11-24T08:15:16,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/35c82d91fdeb446c83c09772a7e6aa18 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/35c82d91fdeb446c83c09772a7e6aa18 2024-11-24T08:15:16,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/6c2bc7eadcb74e52ae34e183e3bc6840 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/6c2bc7eadcb74e52ae34e183e3bc6840 2024-11-24T08:15:16,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/705de8ccf1bb45619101c77ff260fd50 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/705de8ccf1bb45619101c77ff260fd50 2024-11-24T08:15:16,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/557052710fe24908bf05359a1a708f6d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/557052710fe24908bf05359a1a708f6d 2024-11-24T08:15:16,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/b5318c7600054d449b51603b5b6d88e3 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/b5318c7600054d449b51603b5b6d88e3 2024-11-24T08:15:16,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/b9c3f49d70b041c6901a34fb84d6b9cb to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/b9c3f49d70b041c6901a34fb84d6b9cb 2024-11-24T08:15:16,586 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/a94240476afe403ca6ec3134434fbab8 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/a94240476afe403ca6ec3134434fbab8 2024-11-24T08:15:16,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/16b96be86774407e95f0acd91705c0e1 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/16b96be86774407e95f0acd91705c0e1 2024-11-24T08:15:16,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/9898ee77872b4642b80f3534d949dd31 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/9898ee77872b4642b80f3534d949dd31 2024-11-24T08:15:16,588 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/de88a25d875845e8bd7206b5a7989271 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/de88a25d875845e8bd7206b5a7989271 2024-11-24T08:15:16,589 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/f6af2ae716fd4e6db73326800f690228 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/f6af2ae716fd4e6db73326800f690228 2024-11-24T08:15:16,590 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/7c26f67b25ce4621b2cc67c6bc8f00e4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/7c26f67b25ce4621b2cc67c6bc8f00e4 2024-11-24T08:15:16,591 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/070fe0ea694e4d9cb1c08a5644d4dbe5 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/070fe0ea694e4d9cb1c08a5644d4dbe5 2024-11-24T08:15:16,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/2ce92308fa6e422ca5697aadc4dfe2da to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/2ce92308fa6e422ca5697aadc4dfe2da 2024-11-24T08:15:16,593 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/e060bab8704e47ff8f38d3d62cd4e339 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/e060bab8704e47ff8f38d3d62cd4e339 2024-11-24T08:15:16,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/68d1cf966b534c2199ebcf162b77d618 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/68d1cf966b534c2199ebcf162b77d618 2024-11-24T08:15:16,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/21e14c9d97c04d38be245a7b8a2b1134 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/21e14c9d97c04d38be245a7b8a2b1134 2024-11-24T08:15:16,596 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/779bffe7a64a40e992bd8def3528042a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/779bffe7a64a40e992bd8def3528042a 2024-11-24T08:15:16,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/bf03bf2091f142e987cb156b8136bd7f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/bf03bf2091f142e987cb156b8136bd7f 2024-11-24T08:15:16,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/bcc8734daadb4dba88fce42d09cb323d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/bcc8734daadb4dba88fce42d09cb323d 2024-11-24T08:15:16,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/c329aeb80559444f9f9f898b766c84af to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/c329aeb80559444f9f9f898b766c84af 2024-11-24T08:15:16,599 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/68d413929b0b4541aed20b05933d9919 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/68d413929b0b4541aed20b05933d9919 2024-11-24T08:15:16,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/1dc9b6264f9a480ca75aa01fa20e539f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/1dc9b6264f9a480ca75aa01fa20e539f 2024-11-24T08:15:16,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/c3e2854d2e0046ceb4a4398a60d6c31b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/c3e2854d2e0046ceb4a4398a60d6c31b 2024-11-24T08:15:16,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/36c01684f9124a2bacf961fc386077ff to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/36c01684f9124a2bacf961fc386077ff 2024-11-24T08:15:16,602 DEBUG [StoreCloser-TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/0f623e55598e41d385315f6ede328a3f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/0f623e55598e41d385315f6ede328a3f 2024-11-24T08:15:16,607 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/recovered.edits/751.seqid, newMaxSeqId=751, maxSeqId=1 2024-11-24T08:15:16,608 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. 2024-11-24T08:15:16,608 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 4e438c4f9c38da2a2238da06f655542f: Waiting for close lock at 1732436115273Running coprocessor pre-close hooks at 1732436115273Disabling compacts and flushes for region at 1732436115273Disabling writes for close at 1732436115273Obtaining lock to block concurrent updates at 1732436115273Preparing flush snapshotting stores in 4e438c4f9c38da2a2238da06f655542f at 1732436115273Finished memstore snapshotting TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732436115274 (+1 ms)Flushing stores of TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f. at 1732436115274Flushing 4e438c4f9c38da2a2238da06f655542f/A: creating writer at 1732436115274Flushing 4e438c4f9c38da2a2238da06f655542f/A: appending metadata at 1732436115277 (+3 ms)Flushing 4e438c4f9c38da2a2238da06f655542f/A: closing flushed file at 1732436115277Flushing 4e438c4f9c38da2a2238da06f655542f/B: creating writer at 1732436115685 (+408 ms)Flushing 4e438c4f9c38da2a2238da06f655542f/B: appending metadata at 1732436115687 (+2 ms)Flushing 4e438c4f9c38da2a2238da06f655542f/B: closing flushed file at 1732436115687Flushing 4e438c4f9c38da2a2238da06f655542f/C: creating writer at 1732436116096 (+409 ms)Flushing 4e438c4f9c38da2a2238da06f655542f/C: appending metadata at 1732436116098 (+2 ms)Flushing 4e438c4f9c38da2a2238da06f655542f/C: closing flushed file at 1732436116098Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58876e81: reopening flushed file at 1732436116509 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c1ebd82: reopening flushed file at 1732436116513 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4350429d: reopening flushed file at 1732436116516 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4e438c4f9c38da2a2238da06f655542f in 1247ms, sequenceid=748, compaction requested=false at 1732436116520 (+4 ms)Writing region close event to WAL at 1732436116604 (+84 ms)Running coprocessor post-close hooks at 1732436116607 (+3 ms)Closed at 1732436116608 (+1 ms) 2024-11-24T08:15:16,609 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:15:16,610 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=4e438c4f9c38da2a2238da06f655542f, regionState=CLOSED 2024-11-24T08:15:16,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:15:16,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-24T08:15:16,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 4e438c4f9c38da2a2238da06f655542f, server=30c28c82771d,45087,1732436016722 in 1.4910 sec 2024-11-24T08:15:16,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-24T08:15:16,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4e438c4f9c38da2a2238da06f655542f, UNASSIGN in 1.4960 sec 2024-11-24T08:15:16,617 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-24T08:15:16,617 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4990 sec 2024-11-24T08:15:16,618 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436116618"}]},"ts":"1732436116618"} 2024-11-24T08:15:16,619 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T08:15:16,619 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T08:15:16,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5070 sec 2024-11-24T08:15:17,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-24T08:15:17,260 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:17,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T08:15:17,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:17,262 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-24T08:15:17,262 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:17,265 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:15:17,267 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/recovered.edits] 2024-11-24T08:15:17,269 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/62245d7797da49c4b15bbd6f0c4d290d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/62245d7797da49c4b15bbd6f0c4d290d 2024-11-24T08:15:17,270 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/bc1769bed8a34fc09057feafe77ecf6b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/A/bc1769bed8a34fc09057feafe77ecf6b 2024-11-24T08:15:17,272 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/7063859cae2c4d19a1424dd9b5d76fcf to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/7063859cae2c4d19a1424dd9b5d76fcf 2024-11-24T08:15:17,273 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/de5ad085f39f41d8a28bb326641dcdef to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/B/de5ad085f39f41d8a28bb326641dcdef 2024-11-24T08:15:17,275 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/22b390fb3036423cbad325649d45fd99 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/22b390fb3036423cbad325649d45fd99 2024-11-24T08:15:17,276 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/d05556f682864f188abb9ebf7baa46a7 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/C/d05556f682864f188abb9ebf7baa46a7 2024-11-24T08:15:17,279 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/recovered.edits/751.seqid to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f/recovered.edits/751.seqid 2024-11-24T08:15:17,279 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/4e438c4f9c38da2a2238da06f655542f 2024-11-24T08:15:17,279 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T08:15:17,282 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:17,285 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T08:15:17,287 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T08:15:17,288 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:17,288 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T08:15:17,288 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732436117288"}]},"ts":"9223372036854775807"} 2024-11-24T08:15:17,290 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T08:15:17,290 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4e438c4f9c38da2a2238da06f655542f, NAME => 'TestAcidGuarantees,,1732436083572.4e438c4f9c38da2a2238da06f655542f.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T08:15:17,290 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T08:15:17,290 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732436117290"}]},"ts":"9223372036854775807"} 2024-11-24T08:15:17,291 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T08:15:17,292 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:17,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-24T08:15:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-24T08:15:17,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T08:15:17,371 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:17,380 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248 (was 252), OpenFileDescriptor=453 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=314 (was 383), ProcessCount=11 (was 11), AvailableMemoryMB=6798 (was 7413) 2024-11-24T08:15:17,388 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=314, ProcessCount=11, AvailableMemoryMB=6798 2024-11-24T08:15:17,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T08:15:17,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:15:17,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:17,391 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T08:15:17,391 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:17,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-24T08:15:17,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T08:15:17,392 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T08:15:17,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742160_1336 (size=1061) 2024-11-24T08:15:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T08:15:17,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T08:15:17,798 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6 2024-11-24T08:15:17,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742161_1337 (size=53) 2024-11-24T08:15:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T08:15:18,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:15:18,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 36db097a55561465c9ef689006e62ac4, disabling compactions & flushes 2024-11-24T08:15:18,204 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:18,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:18,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. after waiting 0 ms 2024-11-24T08:15:18,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:18,204 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:18,204 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 36db097a55561465c9ef689006e62ac4: Waiting for close lock at 1732436118204Disabling compacts and flushes for region at 1732436118204Disabling writes for close at 1732436118204Writing region close event to WAL at 1732436118204Closed at 1732436118204 2024-11-24T08:15:18,205 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T08:15:18,205 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732436118205"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732436118205"}]},"ts":"1732436118205"} 2024-11-24T08:15:18,207 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T08:15:18,207 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T08:15:18,208 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436118207"}]},"ts":"1732436118207"} 2024-11-24T08:15:18,209 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T08:15:18,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, ASSIGN}] 2024-11-24T08:15:18,210 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, ASSIGN 2024-11-24T08:15:18,211 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, ASSIGN; state=OFFLINE, location=30c28c82771d,45087,1732436016722; forceNewPlan=false, retain=false 2024-11-24T08:15:18,362 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=36db097a55561465c9ef689006e62ac4, regionState=OPENING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:15:18,365 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, ASSIGN because future has completed 2024-11-24T08:15:18,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:15:18,520 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:18,520 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:15:18,520 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:18,521 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:15:18,521 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:18,521 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:18,524 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:18,525 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:15:18,525 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36db097a55561465c9ef689006e62ac4 columnFamilyName A 2024-11-24T08:15:18,525 DEBUG [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:18,526 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.HStore(327): Store=36db097a55561465c9ef689006e62ac4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:15:18,526 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:18,527 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:15:18,527 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36db097a55561465c9ef689006e62ac4 columnFamilyName B 2024-11-24T08:15:18,527 DEBUG [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:18,527 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.HStore(327): Store=36db097a55561465c9ef689006e62ac4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:15:18,527 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:18,528 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:15:18,528 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36db097a55561465c9ef689006e62ac4 columnFamilyName C 2024-11-24T08:15:18,528 DEBUG [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:18,529 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.HStore(327): Store=36db097a55561465c9ef689006e62ac4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:15:18,529 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:18,529 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:18,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T08:15:18,530 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:18,530 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:18,531 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:18,531 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:18,531 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T08:15:18,532 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:18,534 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:15:18,534 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 36db097a55561465c9ef689006e62ac4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59726337, jitterRate=-0.11000822484493256}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T08:15:18,534 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:18,535 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 36db097a55561465c9ef689006e62ac4: Running coprocessor pre-open hook at 1732436118521Writing region info on filesystem at 1732436118521Initializing all the Stores at 1732436118521Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436118521Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436118524 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436118524Cleaning up temporary data from old regions at 1732436118531 (+7 ms)Running coprocessor post-open hooks at 1732436118534 (+3 ms)Region opened successfully at 1732436118535 (+1 ms) 2024-11-24T08:15:18,536 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., pid=95, masterSystemTime=1732436118517 2024-11-24T08:15:18,537 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:18,538 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:18,538 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=36db097a55561465c9ef689006e62ac4, regionState=OPEN, openSeqNum=2, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:15:18,539 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:15:18,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-24T08:15:18,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 in 175 msec 2024-11-24T08:15:18,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-24T08:15:18,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, ASSIGN in 333 msec 2024-11-24T08:15:18,544 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T08:15:18,544 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436118544"}]},"ts":"1732436118544"} 2024-11-24T08:15:18,546 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T08:15:18,546 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T08:15:18,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1570 sec 2024-11-24T08:15:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-24T08:15:19,541 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:19,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283b5885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:19,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:19,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:19,552 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1880dbd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:19,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:19,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:19,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:19,554 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:19,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1305d3ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:19,555 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:19,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:19,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:19,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:19,559 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T08:15:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 30c28c82771d,38469,1732436015766 2024-11-24T08:15:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2ff4c25 2024-11-24T08:15:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T08:15:19,561 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T08:15:19,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T08:15:19,563 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:15:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:19,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742162_1338 (size=1097) 2024-11-24T08:15:20,028 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T08:15:20,028 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T08:15:20,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T08:15:20,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, REOPEN/MOVE}] 2024-11-24T08:15:20,035 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, REOPEN/MOVE 2024-11-24T08:15:20,036 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=36db097a55561465c9ef689006e62ac4, regionState=CLOSING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:15:20,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, REOPEN/MOVE because future has completed 2024-11-24T08:15:20,038 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T08:15:20,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:15:20,191 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,191 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T08:15:20,191 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 36db097a55561465c9ef689006e62ac4, disabling compactions & flushes 2024-11-24T08:15:20,191 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:20,191 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:20,191 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. after waiting 0 ms 2024-11-24T08:15:20,191 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:20,196 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T08:15:20,196 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:20,196 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 36db097a55561465c9ef689006e62ac4: Waiting for close lock at 1732436120191Running coprocessor pre-close hooks at 1732436120191Disabling compacts and flushes for region at 1732436120191Disabling writes for close at 1732436120191Writing region close event to WAL at 1732436120192 (+1 ms)Running coprocessor post-close hooks at 1732436120196 (+4 ms)Closed at 1732436120196 2024-11-24T08:15:20,196 WARN [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: 36db097a55561465c9ef689006e62ac4 to self. 2024-11-24T08:15:20,199 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,199 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=36db097a55561465c9ef689006e62ac4, regionState=CLOSED 2024-11-24T08:15:20,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:15:20,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-24T08:15:20,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 in 164 msec 2024-11-24T08:15:20,204 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, REOPEN/MOVE; state=CLOSED, location=30c28c82771d,45087,1732436016722; forceNewPlan=false, retain=true 2024-11-24T08:15:20,355 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=36db097a55561465c9ef689006e62ac4, regionState=OPENING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:15:20,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, REOPEN/MOVE because future has completed 2024-11-24T08:15:20,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:15:20,511 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:20,511 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:15:20,512 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,512 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:15:20,512 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,512 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,513 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,514 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:15:20,514 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36db097a55561465c9ef689006e62ac4 columnFamilyName A 2024-11-24T08:15:20,515 DEBUG [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:20,515 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.HStore(327): Store=36db097a55561465c9ef689006e62ac4/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:15:20,516 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,516 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:15:20,516 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36db097a55561465c9ef689006e62ac4 columnFamilyName B 2024-11-24T08:15:20,516 DEBUG [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:20,517 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.HStore(327): Store=36db097a55561465c9ef689006e62ac4/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:15:20,517 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,517 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:15:20,517 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 36db097a55561465c9ef689006e62ac4 columnFamilyName C 2024-11-24T08:15:20,517 DEBUG [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:20,518 INFO [StoreOpener-36db097a55561465c9ef689006e62ac4-1 {}] regionserver.HStore(327): Store=36db097a55561465c9ef689006e62ac4/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:15:20,518 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:20,518 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,518 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,519 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,519 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,520 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,520 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T08:15:20,521 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,521 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 36db097a55561465c9ef689006e62ac4; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65543949, jitterRate=-0.023319050669670105}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T08:15:20,521 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,522 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 36db097a55561465c9ef689006e62ac4: Running coprocessor pre-open hook at 1732436120512Writing region info on filesystem at 1732436120512Initializing all the Stores at 1732436120512Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436120512Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436120513 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436120513Cleaning up temporary data from old regions at 1732436120520 (+7 ms)Running coprocessor post-open hooks at 1732436120521 (+1 ms)Region opened successfully at 1732436120522 (+1 ms) 2024-11-24T08:15:20,522 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., pid=100, masterSystemTime=1732436120508 2024-11-24T08:15:20,524 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:20,524 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:20,524 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=36db097a55561465c9ef689006e62ac4, regionState=OPEN, openSeqNum=5, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:15:20,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:15:20,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-24T08:15:20,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 in 170 msec 2024-11-24T08:15:20,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-24T08:15:20,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, REOPEN/MOVE in 493 msec 2024-11-24T08:15:20,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-24T08:15:20,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 498 msec 2024-11-24T08:15:20,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 968 msec 2024-11-24T08:15:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-24T08:15:20,533 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:20,534 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf7f63f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,534 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:20,534 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:20,534 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f247131, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:20,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:20,536 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:20,536 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bb4dc28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4600746, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,537 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:20,537 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:20,537 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:20,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6baed8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:20,539 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:20,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15b8f0d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6447def3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:20,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:20,540 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:20,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e9017fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:20,541 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:20,542 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1efe1fff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,542 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5345722e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,542 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:20,542 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:20,543 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7495f07f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:20,544 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e93bc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dd3a90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:20,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:20,546 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@345864a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:20,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:20,547 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:20,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bc1eac0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eba3e2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:20,548 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:20,548 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:20,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24f1f3fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:20,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:20,549 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:20,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71b06369, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,550 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79570776, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,550 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:20,550 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:20,551 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29ed9ba3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:20,552 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:20,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339451b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43065a6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,553 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:20,553 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:20,553 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59dca9c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:20,554 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:20,555 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45530, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:20,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c15b51e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,556 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bcd92c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,556 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:20,556 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:20,556 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:20,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:20,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:20,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2473bbbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:20,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:20,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:20,558 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:20,558 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1874021, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b779047, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:20,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:20,559 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7021fa63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:20,560 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1958ba98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:20,561 DEBUG [Thread-1547 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:20,561 DEBUG [Thread-1558 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:20,561 DEBUG [Thread-1560 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:20,561 DEBUG [Thread-1549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:20,562 DEBUG [Thread-1552 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:20,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:20,562 DEBUG [Thread-1556 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:20,562 DEBUG [Thread-1554 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:20,563 DEBUG [Thread-1545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-24T08:15:20,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:20,564 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:20,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:20,564 DEBUG [Thread-1552 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:20,564 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:20,564 DEBUG [Thread-1552 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:20,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:20,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:20,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:20,566 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:20,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T08:15:20,566 DEBUG [Thread-1541 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:20,567 DEBUG [Thread-1543 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:20,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:20,567 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:20,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:20,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:20,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:15:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:15:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:15:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:20,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:15:20,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:15:20,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:20,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:15:20,570 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:15:20,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:15:20,571 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:20,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:15:20,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:15:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:20,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:15:20,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:20,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:20,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:20,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:20,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:20,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:20,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b9069e4eb9ca403aaacca1d2846833a1_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436120576/Put/seqid=0 2024-11-24T08:15:20,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742163_1339 (size=14594) 2024-11-24T08:15:20,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:20,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732436120601, max timestamp=1732436120627Num uniques 859; 2024-11-24T08:15:20,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732436120601, max timestamp=1732436120627Num uniques 859; 2024-11-24T08:15:20,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732436120601, max timestamp=1732436120627Num uniques 859; 2024-11-24T08:15:20,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:15:20,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:15:20,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-24T08:15:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436180618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T08:15:20,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436180665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:20,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:20,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436180666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:20,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:20,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436180668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:20,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:20,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:20,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60138 deadline: 1732436180680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:20,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:20,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T08:15:20,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:20,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:20,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:20,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:20,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:20,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436180739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:20,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:20,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436180790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436180790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:20,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:20,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436180790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:20,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:20,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:20,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:20,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60138 deadline: 1732436180790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:20,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:20,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T08:15:20,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:20,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:20,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:20,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:20,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:20,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T08:15:20,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436180960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:20,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,014 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:21,018 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b9069e4eb9ca403aaacca1d2846833a1_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b9069e4eb9ca403aaacca1d2846833a1_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:21,019 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/6505cb74fead408298f4e30ed5a9e9c4, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:21,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436181010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:21,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/6505cb74fead408298f4e30ed5a9e9c4 is 175, key is test_row_0/A:col10/1732436120576/Put/seqid=0 2024-11-24T08:15:21,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436181010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:21,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436181010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:21,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60138 deadline: 1732436181010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:21,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742164_1340 (size=39549) 2024-11-24T08:15:21,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T08:15:21,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:21,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:21,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:21,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T08:15:21,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:21,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:21,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:21,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T08:15:21,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436181279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:21,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436181330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:21,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T08:15:21,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436181330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:21,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:21,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60138 deadline: 1732436181330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436181330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,429 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/6505cb74fead408298f4e30ed5a9e9c4 2024-11-24T08:15:21,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/758c2f05060843b5acf8a892ed9d365d is 50, key is test_row_0/B:col10/1732436120576/Put/seqid=0 2024-11-24T08:15:21,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742165_1341 (size=12001) 2024-11-24T08:15:21,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T08:15:21,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:21,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:21,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:21,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T08:15:21,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:21,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:21,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:21,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T08:15:21,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T08:15:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:21,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436181790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:21,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436181850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436181850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:21,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60138 deadline: 1732436181850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436181850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:21,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:21,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/758c2f05060843b5acf8a892ed9d365d 2024-11-24T08:15:21,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/fff4b2bd0cf64988a7b12b6559a7eb56 is 50, key is test_row_0/C:col10/1732436120576/Put/seqid=0 2024-11-24T08:15:21,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742166_1342 (size=12001) 2024-11-24T08:15:21,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T08:15:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:21,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:22,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T08:15:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:22,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:22,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:22,130 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T08:15:22,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T08:15:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:22,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:22,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:22,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/fff4b2bd0cf64988a7b12b6559a7eb56 2024-11-24T08:15:22,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/6505cb74fead408298f4e30ed5a9e9c4 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/6505cb74fead408298f4e30ed5a9e9c4 2024-11-24T08:15:22,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/6505cb74fead408298f4e30ed5a9e9c4, entries=200, sequenceid=14, filesize=38.6 K 2024-11-24T08:15:22,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/758c2f05060843b5acf8a892ed9d365d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/758c2f05060843b5acf8a892ed9d365d 2024-11-24T08:15:22,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/758c2f05060843b5acf8a892ed9d365d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-24T08:15:22,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/fff4b2bd0cf64988a7b12b6559a7eb56 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/fff4b2bd0cf64988a7b12b6559a7eb56 2024-11-24T08:15:22,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/fff4b2bd0cf64988a7b12b6559a7eb56, entries=150, sequenceid=14, filesize=11.7 K 2024-11-24T08:15:22,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 36db097a55561465c9ef689006e62ac4 in 1726ms, sequenceid=14, compaction requested=false 2024-11-24T08:15:22,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:22,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-24T08:15:22,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:22,409 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-24T08:15:22,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:22,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:22,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:22,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:22,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:22,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:22,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bb28332e1cbe4ab7baf56fc101c6cc2e_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436120678/Put/seqid=0 2024-11-24T08:15:22,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742167_1343 (size=12154) 2024-11-24T08:15:22,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:22,425 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bb28332e1cbe4ab7baf56fc101c6cc2e_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bb28332e1cbe4ab7baf56fc101c6cc2e_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:22,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/55de1a9d081d4ccea9c4126d530a930b, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:22,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/55de1a9d081d4ccea9c4126d530a930b is 175, key is test_row_0/A:col10/1732436120678/Put/seqid=0 2024-11-24T08:15:22,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742168_1344 (size=30955) 2024-11-24T08:15:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T08:15:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:22,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:22,836 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/55de1a9d081d4ccea9c4126d530a930b 2024-11-24T08:15:22,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/c93a8babb07f4a0db5aeb21ea96d8a6d is 50, key is test_row_0/B:col10/1732436120678/Put/seqid=0 2024-11-24T08:15:22,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742169_1345 (size=12001) 2024-11-24T08:15:22,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:22,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436182860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:22,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:22,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436182870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:22,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:22,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436182870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:22,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60138 deadline: 1732436182870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:22,880 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 36db097a55561465c9ef689006e62ac4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:22,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:22,881 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 36db097a55561465c9ef689006e62ac4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436182870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:22,881 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 36db097a55561465c9ef689006e62ac4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:22,882 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in 36db097a55561465c9ef689006e62ac4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:22,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:22,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436182979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:22,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:23,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436183190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:23,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:23,250 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/c93a8babb07f4a0db5aeb21ea96d8a6d 2024-11-24T08:15:23,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/b7a99bd56ea247d8915beafc672b5806 is 50, key is test_row_0/C:col10/1732436120678/Put/seqid=0 2024-11-24T08:15:23,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742170_1346 (size=12001) 2024-11-24T08:15:23,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436183499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:23,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:23,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:23,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:23,676 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/b7a99bd56ea247d8915beafc672b5806 2024-11-24T08:15:23,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/55de1a9d081d4ccea9c4126d530a930b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/55de1a9d081d4ccea9c4126d530a930b 2024-11-24T08:15:23,684 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/55de1a9d081d4ccea9c4126d530a930b, entries=150, sequenceid=50, filesize=30.2 K 2024-11-24T08:15:23,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/c93a8babb07f4a0db5aeb21ea96d8a6d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/c93a8babb07f4a0db5aeb21ea96d8a6d 2024-11-24T08:15:23,691 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/c93a8babb07f4a0db5aeb21ea96d8a6d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-24T08:15:23,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/b7a99bd56ea247d8915beafc672b5806 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/b7a99bd56ea247d8915beafc672b5806 2024-11-24T08:15:23,695 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/b7a99bd56ea247d8915beafc672b5806, entries=150, sequenceid=50, filesize=11.7 K 2024-11-24T08:15:23,697 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 36db097a55561465c9ef689006e62ac4 in 1288ms, sequenceid=50, compaction requested=false 2024-11-24T08:15:23,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:23,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:23,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-24T08:15:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-24T08:15:23,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-24T08:15:23,700 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1320 sec 2024-11-24T08:15:23,702 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 3.1390 sec 2024-11-24T08:15:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:24,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T08:15:24,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:24,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:24,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:24,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:24,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:24,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:24,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243f750689bcef43799710ba937f3bd7a6_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436122858/Put/seqid=0 2024-11-24T08:15:24,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742171_1347 (size=14594) 2024-11-24T08:15:24,034 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:24,038 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411243f750689bcef43799710ba937f3bd7a6_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243f750689bcef43799710ba937f3bd7a6_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:24,039 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/ad07df2b09e144f8935a6e769642bcb0, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:24,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/ad07df2b09e144f8935a6e769642bcb0 is 175, key is test_row_0/A:col10/1732436122858/Put/seqid=0 2024-11-24T08:15:24,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742172_1348 (size=39549) 2024-11-24T08:15:24,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436124123, max timestamp=1732436124189Num uniques 859; 2024-11-24T08:15:24,193 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:15:24,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436124123, max timestamp=1732436124189Num uniques 859; 2024-11-24T08:15:24,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:15:24,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436124123, max timestamp=1732436124189Num uniques 859; 2024-11-24T08:15:24,194 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:15:24,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436184361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:24,444 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/ad07df2b09e144f8935a6e769642bcb0 2024-11-24T08:15:24,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/83e611a8b8a941e7a0dc2c2fc9f91198 is 50, key is test_row_0/B:col10/1732436122858/Put/seqid=0 2024-11-24T08:15:24,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742173_1349 (size=12001) 2024-11-24T08:15:24,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436184480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:24,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:24,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:24,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:24,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436184699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-24T08:15:24,720 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:24,721 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-24T08:15:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T08:15:24,722 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:24,723 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:24,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T08:15:24,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/83e611a8b8a941e7a0dc2c2fc9f91198 2024-11-24T08:15:24,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/13ffa5a706d646149ae70d0e1114d83b is 50, key is test_row_0/C:col10/1732436122858/Put/seqid=0 2024-11-24T08:15:24,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T08:15:24,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:24,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:24,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:24,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:24,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:24,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742174_1350 (size=12001) 2024-11-24T08:15:24,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/13ffa5a706d646149ae70d0e1114d83b 2024-11-24T08:15:24,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/ad07df2b09e144f8935a6e769642bcb0 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/ad07df2b09e144f8935a6e769642bcb0 2024-11-24T08:15:24,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/ad07df2b09e144f8935a6e769642bcb0, entries=200, sequenceid=61, filesize=38.6 K 2024-11-24T08:15:24,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/83e611a8b8a941e7a0dc2c2fc9f91198 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/83e611a8b8a941e7a0dc2c2fc9f91198 2024-11-24T08:15:24,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/83e611a8b8a941e7a0dc2c2fc9f91198, entries=150, sequenceid=61, filesize=11.7 K 2024-11-24T08:15:24,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/13ffa5a706d646149ae70d0e1114d83b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/13ffa5a706d646149ae70d0e1114d83b 2024-11-24T08:15:24,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436184890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:24,908 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 36db097a55561465c9ef689006e62ac4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:24,909 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:24,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/13ffa5a706d646149ae70d0e1114d83b, entries=150, sequenceid=61, filesize=11.7 K 2024-11-24T08:15:24,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 36db097a55561465c9ef689006e62ac4 in 890ms, sequenceid=61, compaction requested=true 2024-11-24T08:15:24,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:24,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:24,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:24,914 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:24,915 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:24,915 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/A is initiating minor compaction (all files) 2024-11-24T08:15:24,915 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/A in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:24,915 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:24,915 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/6505cb74fead408298f4e30ed5a9e9c4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/55de1a9d081d4ccea9c4126d530a930b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/ad07df2b09e144f8935a6e769642bcb0] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=107.5 K 2024-11-24T08:15:24,915 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:24,915 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/6505cb74fead408298f4e30ed5a9e9c4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/55de1a9d081d4ccea9c4126d530a930b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/ad07df2b09e144f8935a6e769642bcb0] 2024-11-24T08:15:24,916 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6505cb74fead408298f4e30ed5a9e9c4, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732436120573 2024-11-24T08:15:24,916 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55de1a9d081d4ccea9c4126d530a930b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732436120647 2024-11-24T08:15:24,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:24,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:24,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:24,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:24,918 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:24,918 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad07df2b09e144f8935a6e769642bcb0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732436122822 2024-11-24T08:15:24,918 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/B is initiating minor compaction (all files) 2024-11-24T08:15:24,919 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/B in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:24,919 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/758c2f05060843b5acf8a892ed9d365d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/c93a8babb07f4a0db5aeb21ea96d8a6d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/83e611a8b8a941e7a0dc2c2fc9f91198] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=35.2 K 2024-11-24T08:15:24,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:24,919 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 758c2f05060843b5acf8a892ed9d365d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732436120573 2024-11-24T08:15:24,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-24T08:15:24,920 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting c93a8babb07f4a0db5aeb21ea96d8a6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732436120647 2024-11-24T08:15:24,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732436124224, max timestamp=1732436124919Num uniques 1018; 2024-11-24T08:15:24,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T08:15:24,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732436124224, max timestamp=1732436124919Num uniques 1018; 2024-11-24T08:15:24,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T08:15:24,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732436124224, max timestamp=1732436124919Num uniques 1018; 2024-11-24T08:15:24,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-24T08:15:24,924 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 83e611a8b8a941e7a0dc2c2fc9f91198, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732436122833 2024-11-24T08:15:24,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:24,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:24,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:24,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:24,925 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:24,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:24,932 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:24,951 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#B#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:24,951 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/88346d2a7c754d32ba570b6d1f246ade is 50, key is test_row_0/B:col10/1732436122858/Put/seqid=0 2024-11-24T08:15:24,961 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112408e8c7d2d00a4352b1e0025b7700dfa0_36db097a55561465c9ef689006e62ac4 store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:24,963 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112408e8c7d2d00a4352b1e0025b7700dfa0_36db097a55561465c9ef689006e62ac4, store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:24,963 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112408e8c7d2d00a4352b1e0025b7700dfa0_36db097a55561465c9ef689006e62ac4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:24,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112404311178ca4549a3a1163c4491b906fd_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436124224/Put/seqid=0 2024-11-24T08:15:24,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742175_1351 (size=12104) 2024-11-24T08:15:25,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742176_1352 (size=4469) 2024-11-24T08:15:25,005 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/88346d2a7c754d32ba570b6d1f246ade as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/88346d2a7c754d32ba570b6d1f246ade 2024-11-24T08:15:25,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742177_1353 (size=14594) 2024-11-24T08:15:25,006 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:25,010 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112404311178ca4549a3a1163c4491b906fd_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112404311178ca4549a3a1163c4491b906fd_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:25,011 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/e15f7fe39f8c4c18a92f71f9022537bc, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:25,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/e15f7fe39f8c4c18a92f71f9022537bc is 175, key is test_row_0/A:col10/1732436124224/Put/seqid=0 2024-11-24T08:15:25,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742178_1354 (size=39549) 2024-11-24T08:15:25,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T08:15:25,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:25,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T08:15:25,046 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36db097a55561465c9ef689006e62ac4/B of 36db097a55561465c9ef689006e62ac4 into 88346d2a7c754d32ba570b6d1f246ade(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:25,047 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:25,047 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/B, priority=13, startTime=1732436124915; duration=0sec 2024-11-24T08:15:25,047 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:25,047 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:B 2024-11-24T08:15:25,047 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:25,048 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:25,048 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/C is initiating minor compaction (all files) 2024-11-24T08:15:25,049 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/C in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,049 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/fff4b2bd0cf64988a7b12b6559a7eb56, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/b7a99bd56ea247d8915beafc672b5806, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/13ffa5a706d646149ae70d0e1114d83b] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=35.2 K 2024-11-24T08:15:25,049 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting fff4b2bd0cf64988a7b12b6559a7eb56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732436120573 2024-11-24T08:15:25,049 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting b7a99bd56ea247d8915beafc672b5806, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732436120647 2024-11-24T08:15:25,050 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 13ffa5a706d646149ae70d0e1114d83b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732436122833 2024-11-24T08:15:25,057 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#C#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:25,058 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/79e8433404c5421e9ddf114983ed173d is 50, key is test_row_0/C:col10/1732436122858/Put/seqid=0 2024-11-24T08:15:25,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742179_1355 (size=12104) 2024-11-24T08:15:25,073 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/79e8433404c5421e9ddf114983ed173d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/79e8433404c5421e9ddf114983ed173d 2024-11-24T08:15:25,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732436124952, max timestamp=1732436125073Num uniques 859; 2024-11-24T08:15:25,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732436124952, max timestamp=1732436125073Num uniques 859; 2024-11-24T08:15:25,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T08:15:25,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T08:15:25,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732436124952, max timestamp=1732436125073Num uniques 859; 2024-11-24T08:15:25,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T08:15:25,087 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36db097a55561465c9ef689006e62ac4/C of 36db097a55561465c9ef689006e62ac4 into 79e8433404c5421e9ddf114983ed173d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:25,087 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:25,087 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/C, priority=13, startTime=1732436124917; duration=0sec 2024-11-24T08:15:25,088 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:25,088 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:C 2024-11-24T08:15:25,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732436125124, max timestamp=1732436125156Num uniques 1168; 2024-11-24T08:15:25,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732436125124, max timestamp=1732436125156Num uniques 1168; 2024-11-24T08:15:25,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=12268, suffix heap size=149440,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=268 2024-11-24T08:15:25,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=12268, suffix heap size=149440,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=268 2024-11-24T08:15:25,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1732436125124, max timestamp=1732436125156Num uniques 1168; 2024-11-24T08:15:25,164 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=12268, suffix heap size=149440,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=268 2024-11-24T08:15:25,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T08:15:25,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:25,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:25,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436185214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:25,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:25,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732436125176, max timestamp=1732436125228Num uniques 1127; 2024-11-24T08:15:25,232 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-24T08:15:25,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1732436125176, max timestamp=1732436125228Num uniques 1127; 2024-11-24T08:15:25,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=12682, suffix heap size=145176,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=277 2024-11-24T08:15:25,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732436125210, max timestamp=1732436125228Num uniques 1127; 2024-11-24T08:15:25,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-24T08:15:25,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732436125285, max timestamp=1732436125325Num uniques 1136; 2024-11-24T08:15:25,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-24T08:15:25,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732436125285, max timestamp=1732436125325Num uniques 1136; 2024-11-24T08:15:25,331 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=10806, suffix heap size=146112,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=236 2024-11-24T08:15:25,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732436125285, max timestamp=1732436125325Num uniques 1036; 2024-11-24T08:15:25,332 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-24T08:15:25,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T08:15:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T08:15:25,403 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#A#compaction#297 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:25,404 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/0ff347a4da4e4778a2d78f196dc8c454 is 175, key is test_row_0/A:col10/1732436122858/Put/seqid=0 2024-11-24T08:15:25,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732436125351, max timestamp=1732436125404Num uniques 1095; 2024-11-24T08:15:25,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-24T08:15:25,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732436125351, max timestamp=1732436125404Num uniques 1095; 2024-11-24T08:15:25,411 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-24T08:15:25,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1732436125351, max timestamp=1732436125404Num uniques 1095; 2024-11-24T08:15:25,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=13510, suffix heap size=141848,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=295 2024-11-24T08:15:25,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742180_1356 (size=31058) 2024-11-24T08:15:25,419 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/0ff347a4da4e4778a2d78f196dc8c454 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/0ff347a4da4e4778a2d78f196dc8c454 2024-11-24T08:15:25,420 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/e15f7fe39f8c4c18a92f71f9022537bc 2024-11-24T08:15:25,430 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36db097a55561465c9ef689006e62ac4/A of 36db097a55561465c9ef689006e62ac4 into 0ff347a4da4e4778a2d78f196dc8c454(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:25,430 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:25,430 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/A, priority=13, startTime=1732436124913; duration=0sec 2024-11-24T08:15:25,430 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:25,430 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:A 2024-11-24T08:15:25,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/4917a478031d44709b0e774d5ea3bc7e is 50, key is test_row_0/B:col10/1732436124224/Put/seqid=0 2024-11-24T08:15:25,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742181_1357 (size=12001) 2024-11-24T08:15:25,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436185470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:25,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T08:15:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436185473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:25,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:25,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436185473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60138 deadline: 1732436185474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436185590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:25,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436185600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:25,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436185600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:25,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60138 deadline: 1732436185600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:25,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,608 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:25,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T08:15:25,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:25,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T08:15:25,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:25,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436185810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:25,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436185810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:25,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436185810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60138 deadline: 1732436185811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:25,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:25,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/4917a478031d44709b0e774d5ea3bc7e 2024-11-24T08:15:25,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/05e184e20746461b92e6a5aef03eef00 is 50, key is test_row_0/C:col10/1732436124224/Put/seqid=0 2024-11-24T08:15:25,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742182_1358 (size=12001) 2024-11-24T08:15:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T08:15:25,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T08:15:25,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:25,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:25,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:25,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:26,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T08:15:26,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:26,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:26,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:26,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:26,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:26,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436186120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:26,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436186130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:26,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:26,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436186130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:26,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:26,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60138 deadline: 1732436186130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:26,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:26,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/05e184e20746461b92e6a5aef03eef00 2024-11-24T08:15:26,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T08:15:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:26,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:26,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:26,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/e15f7fe39f8c4c18a92f71f9022537bc as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/e15f7fe39f8c4c18a92f71f9022537bc 2024-11-24T08:15:26,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/e15f7fe39f8c4c18a92f71f9022537bc, entries=200, sequenceid=99, filesize=38.6 K 2024-11-24T08:15:26,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/4917a478031d44709b0e774d5ea3bc7e as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/4917a478031d44709b0e774d5ea3bc7e 2024-11-24T08:15:26,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/4917a478031d44709b0e774d5ea3bc7e, entries=150, sequenceid=99, filesize=11.7 K 2024-11-24T08:15:26,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/05e184e20746461b92e6a5aef03eef00 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/05e184e20746461b92e6a5aef03eef00 2024-11-24T08:15:26,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/05e184e20746461b92e6a5aef03eef00, entries=150, sequenceid=99, filesize=11.7 K 2024-11-24T08:15:26,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~80.58 KB/82512, currentSize=154.31 KB/158010 for 36db097a55561465c9ef689006e62ac4 in 1352ms, sequenceid=99, compaction requested=false 2024-11-24T08:15:26,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:26,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-24T08:15:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:26,412 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=154.31 KB heapSize=432 KB 2024-11-24T08:15:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:26,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246f0747936e3e48958de5fbbbc1990957_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436125455/Put/seqid=0 2024-11-24T08:15:26,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742183_1359 (size=12304) 2024-11-24T08:15:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:26,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:26,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60138 deadline: 1732436186650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:26,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:26,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436186654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:26,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436186655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436186656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:26,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436186770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:26,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436186770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:26,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436186770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:26,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:26,830 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246f0747936e3e48958de5fbbbc1990957_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246f0747936e3e48958de5fbbbc1990957_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:26,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/c0472b0da4984ddbb5934045e05a8db6, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:26,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/c0472b0da4984ddbb5934045e05a8db6 is 175, key is test_row_0/A:col10/1732436125455/Put/seqid=0 2024-11-24T08:15:26,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742184_1360 (size=31105) 2024-11-24T08:15:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T08:15:26,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436186979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:26,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:26,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436186990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:26,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:26,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436186990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:26,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:26,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:26,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:27,241 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/c0472b0da4984ddbb5934045e05a8db6 2024-11-24T08:15:27,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/4ecca390174a4cc4a1eace1c068f8c5f is 50, key is test_row_0/B:col10/1732436125455/Put/seqid=0 2024-11-24T08:15:27,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742185_1361 (size=12151) 2024-11-24T08:15:27,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436187300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:27,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436187309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:27,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436187310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:27,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:27,667 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/4ecca390174a4cc4a1eace1c068f8c5f 2024-11-24T08:15:27,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/9a313c18899b4cf0928ac19d26cb6a22 is 50, key is test_row_0/C:col10/1732436125455/Put/seqid=0 2024-11-24T08:15:27,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742186_1362 (size=12151) 2024-11-24T08:15:27,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60138 deadline: 1732436187669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:27,679 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in 36db097a55561465c9ef689006e62ac4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:27,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:27,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436187810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:27,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436187830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:27,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436187830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:28,079 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/9a313c18899b4cf0928ac19d26cb6a22 2024-11-24T08:15:28,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/c0472b0da4984ddbb5934045e05a8db6 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/c0472b0da4984ddbb5934045e05a8db6 2024-11-24T08:15:28,088 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/c0472b0da4984ddbb5934045e05a8db6, entries=150, sequenceid=208, filesize=30.4 K 2024-11-24T08:15:28,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/4ecca390174a4cc4a1eace1c068f8c5f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/4ecca390174a4cc4a1eace1c068f8c5f 2024-11-24T08:15:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,093 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/4ecca390174a4cc4a1eace1c068f8c5f, entries=150, sequenceid=208, filesize=11.9 K 2024-11-24T08:15:28,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/9a313c18899b4cf0928ac19d26cb6a22 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/9a313c18899b4cf0928ac19d26cb6a22 2024-11-24T08:15:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,101 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/9a313c18899b4cf0928ac19d26cb6a22, entries=150, sequenceid=208, filesize=11.9 K 2024-11-24T08:15:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,102 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~431.95 KB/442320, currentSize=26.84 KB/27480 for 36db097a55561465c9ef689006e62ac4 in 1689ms, sequenceid=208, compaction requested=true 2024-11-24T08:15:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-24T08:15:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-24T08:15:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-24T08:15:28,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3800 sec 2024-11-24T08:15:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.3850 sec 2024-11-24T08:15:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-24T08:15:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,891 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,892 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-24T08:15:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T08:15:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:28,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:15:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:28,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:28,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:28,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:28,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:28,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:28,897 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,898 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:28,898 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d3ace590b0064d5eae592d1824466ebb_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436128894/Put/seqid=0 2024-11-24T08:15:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:28,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742187_1363 (size=14794) 2024-11-24T08:15:28,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732436128934, max timestamp=1732436128974Num uniques 859; 2024-11-24T08:15:28,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732436128934, max timestamp=1732436128974Num uniques 859; 2024-11-24T08:15:28,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T08:15:28,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T08:15:28,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732436128934, max timestamp=1732436128974Num uniques 859; 2024-11-24T08:15:28,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T08:15:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T08:15:29,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436189013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436189024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436189024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436189026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T08:15:29,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:29,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:29,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:29,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436189140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436189140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436189150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436189150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T08:15:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:29,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:29,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T08:15:29,323 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:29,327 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d3ace590b0064d5eae592d1824466ebb_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d3ace590b0064d5eae592d1824466ebb_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:29,327 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/21667912d2ce4e0488bd5546cf17cefd, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:29,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/21667912d2ce4e0488bd5546cf17cefd is 175, key is test_row_0/A:col10/1732436128894/Put/seqid=0 2024-11-24T08:15:29,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742188_1364 (size=39749) 2024-11-24T08:15:29,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T08:15:29,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:29,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:29,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:29,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436189360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436189360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436189360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436189360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T08:15:29,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:29,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:29,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:29,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T08:15:29,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T08:15:29,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:29,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:29,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:29,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436189680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436189680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436189680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436189680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60138 deadline: 1732436189699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:29,715 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in 36db097a55561465c9ef689006e62ac4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:29,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:29,732 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/21667912d2ce4e0488bd5546cf17cefd 2024-11-24T08:15:29,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/8a09c718601c42abbdcb3683d8fdf898 is 50, key is test_row_0/B:col10/1732436128894/Put/seqid=0 2024-11-24T08:15:29,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742189_1365 (size=12151) 2024-11-24T08:15:29,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/8a09c718601c42abbdcb3683d8fdf898 2024-11-24T08:15:29,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/b706db157c284c6283ab8d2c06d240e5 is 50, key is test_row_0/C:col10/1732436128894/Put/seqid=0 2024-11-24T08:15:29,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742190_1366 (size=12151) 2024-11-24T08:15:29,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T08:15:29,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:29,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:29,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:29,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T08:15:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:29,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T08:15:30,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T08:15:30,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:30,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/b706db157c284c6283ab8d2c06d240e5 2024-11-24T08:15:30,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/21667912d2ce4e0488bd5546cf17cefd as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/21667912d2ce4e0488bd5546cf17cefd 2024-11-24T08:15:30,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/21667912d2ce4e0488bd5546cf17cefd, entries=200, sequenceid=218, filesize=38.8 K 2024-11-24T08:15:30,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/8a09c718601c42abbdcb3683d8fdf898 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/8a09c718601c42abbdcb3683d8fdf898 2024-11-24T08:15:30,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/8a09c718601c42abbdcb3683d8fdf898, entries=150, sequenceid=218, filesize=11.9 K 2024-11-24T08:15:30,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/b706db157c284c6283ab8d2c06d240e5 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/b706db157c284c6283ab8d2c06d240e5 2024-11-24T08:15:30,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/b706db157c284c6283ab8d2c06d240e5, entries=150, sequenceid=218, filesize=11.9 K 2024-11-24T08:15:30,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 36db097a55561465c9ef689006e62ac4 in 1293ms, sequenceid=218, compaction requested=true 2024-11-24T08:15:30,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:30,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:30,190 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:15:30,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:30,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:30,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:30,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:30,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:30,190 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:15:30,194 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:15:30,194 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/B is initiating minor compaction (all files) 2024-11-24T08:15:30,194 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/B in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,194 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/88346d2a7c754d32ba570b6d1f246ade, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/4917a478031d44709b0e774d5ea3bc7e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/4ecca390174a4cc4a1eace1c068f8c5f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/8a09c718601c42abbdcb3683d8fdf898] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=47.3 K 2024-11-24T08:15:30,194 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141461 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:15:30,194 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/A is initiating minor compaction (all files) 2024-11-24T08:15:30,194 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 88346d2a7c754d32ba570b6d1f246ade, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732436122833 2024-11-24T08:15:30,194 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/A in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,195 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/0ff347a4da4e4778a2d78f196dc8c454, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/e15f7fe39f8c4c18a92f71f9022537bc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/c0472b0da4984ddbb5934045e05a8db6, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/21667912d2ce4e0488bd5546cf17cefd] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=138.1 K 2024-11-24T08:15:30,195 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,195 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/0ff347a4da4e4778a2d78f196dc8c454, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/e15f7fe39f8c4c18a92f71f9022537bc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/c0472b0da4984ddbb5934045e05a8db6, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/21667912d2ce4e0488bd5546cf17cefd] 2024-11-24T08:15:30,195 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4917a478031d44709b0e774d5ea3bc7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732436124224 2024-11-24T08:15:30,195 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ff347a4da4e4778a2d78f196dc8c454, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732436122833 2024-11-24T08:15:30,196 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ecca390174a4cc4a1eace1c068f8c5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732436125455 2024-11-24T08:15:30,197 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a09c718601c42abbdcb3683d8fdf898, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732436126653 2024-11-24T08:15:30,198 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e15f7fe39f8c4c18a92f71f9022537bc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732436124224 2024-11-24T08:15:30,198 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0472b0da4984ddbb5934045e05a8db6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732436125455 2024-11-24T08:15:30,198 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21667912d2ce4e0488bd5546cf17cefd, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732436126653 2024-11-24T08:15:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:30,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-24T08:15:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:30,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:30,222 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#B#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:30,223 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/302d4adff8844f5bb78351b37e4cd80c is 50, key is test_row_0/B:col10/1732436128894/Put/seqid=0 2024-11-24T08:15:30,225 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:30,244 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112462085812500f4d8ea5093fe432ac6d7b_36db097a55561465c9ef689006e62ac4 store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:30,247 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112462085812500f4d8ea5093fe432ac6d7b_36db097a55561465c9ef689006e62ac4, store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:30,247 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112462085812500f4d8ea5093fe432ac6d7b_36db097a55561465c9ef689006e62ac4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:30,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:30,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436190242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:30,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:30,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124620aa0dea24d4a69858a1b21303b7a44_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436129023/Put/seqid=0 2024-11-24T08:15:30,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742191_1367 (size=12391) 2024-11-24T08:15:30,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436190245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:30,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436190248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:30,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:30,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436190249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:30,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:30,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T08:15:30,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:30,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,278 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/302d4adff8844f5bb78351b37e4cd80c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/302d4adff8844f5bb78351b37e4cd80c 2024-11-24T08:15:30,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742192_1368 (size=4469) 2024-11-24T08:15:30,281 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#A#compaction#310 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:30,282 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/6160b1dfa0074916968f37076b5ba386 is 175, key is test_row_0/A:col10/1732436128894/Put/seqid=0 2024-11-24T08:15:30,285 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36db097a55561465c9ef689006e62ac4/B of 36db097a55561465c9ef689006e62ac4 into 302d4adff8844f5bb78351b37e4cd80c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:30,285 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:30,285 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/B, priority=12, startTime=1732436130190; duration=0sec 2024-11-24T08:15:30,285 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:30,285 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:B 2024-11-24T08:15:30,285 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:15:30,287 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:15:30,287 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/C is initiating minor compaction (all files) 2024-11-24T08:15:30,287 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/C in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,287 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/79e8433404c5421e9ddf114983ed173d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/05e184e20746461b92e6a5aef03eef00, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/9a313c18899b4cf0928ac19d26cb6a22, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/b706db157c284c6283ab8d2c06d240e5] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=47.3 K 2024-11-24T08:15:30,288 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 79e8433404c5421e9ddf114983ed173d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732436122833 2024-11-24T08:15:30,288 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 05e184e20746461b92e6a5aef03eef00, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1732436124224 2024-11-24T08:15:30,289 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a313c18899b4cf0928ac19d26cb6a22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732436125455 2024-11-24T08:15:30,289 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting b706db157c284c6283ab8d2c06d240e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732436126653 2024-11-24T08:15:30,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742193_1369 (size=14794) 2024-11-24T08:15:30,303 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:30,307 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124620aa0dea24d4a69858a1b21303b7a44_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124620aa0dea24d4a69858a1b21303b7a44_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:30,308 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/c7b115a872e84373802d24fac659bf37, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:30,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/c7b115a872e84373802d24fac659bf37 is 175, key is test_row_0/A:col10/1732436129023/Put/seqid=0 2024-11-24T08:15:30,311 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#C#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:30,311 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/c332eff5186842e5bc9c35a3ff88ab22 is 50, key is test_row_0/C:col10/1732436128894/Put/seqid=0 2024-11-24T08:15:30,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742194_1370 (size=31345) 2024-11-24T08:15:30,322 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/6160b1dfa0074916968f37076b5ba386 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/6160b1dfa0074916968f37076b5ba386 2024-11-24T08:15:30,327 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36db097a55561465c9ef689006e62ac4/A of 36db097a55561465c9ef689006e62ac4 into 6160b1dfa0074916968f37076b5ba386(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:30,327 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:30,327 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/A, priority=12, startTime=1732436130189; duration=0sec 2024-11-24T08:15:30,328 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:30,328 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:A 2024-11-24T08:15:30,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742195_1371 (size=39749) 2024-11-24T08:15:30,335 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/c7b115a872e84373802d24fac659bf37 2024-11-24T08:15:30,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742196_1372 (size=12391) 2024-11-24T08:15:30,343 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/c332eff5186842e5bc9c35a3ff88ab22 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/c332eff5186842e5bc9c35a3ff88ab22 2024-11-24T08:15:30,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/95cf21937e0a4d37b1f45fc7a44cdb56 is 50, key is test_row_0/B:col10/1732436129023/Put/seqid=0 2024-11-24T08:15:30,349 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36db097a55561465c9ef689006e62ac4/C of 36db097a55561465c9ef689006e62ac4 into c332eff5186842e5bc9c35a3ff88ab22(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:30,349 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:30,349 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/C, priority=12, startTime=1732436130190; duration=0sec 2024-11-24T08:15:30,350 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:30,350 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:C 2024-11-24T08:15:30,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436190360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:30,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436190380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:30,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436190380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436190380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:30,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742197_1373 (size=12151) 2024-11-24T08:15:30,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T08:15:30,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:30,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436190571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:30,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T08:15:30,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:30,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436190589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:30,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436190590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:30,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436190590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:30,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:30,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:30,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T08:15:30,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:30,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:30,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/95cf21937e0a4d37b1f45fc7a44cdb56 2024-11-24T08:15:30,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/eef5c221e8b441fb9c2159834280af50 is 50, key is test_row_0/C:col10/1732436129023/Put/seqid=0 2024-11-24T08:15:30,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742198_1374 (size=12151) 2024-11-24T08:15:30,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/eef5c221e8b441fb9c2159834280af50 2024-11-24T08:15:30,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/c7b115a872e84373802d24fac659bf37 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/c7b115a872e84373802d24fac659bf37 2024-11-24T08:15:30,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/c7b115a872e84373802d24fac659bf37, entries=200, sequenceid=254, filesize=38.8 K 2024-11-24T08:15:30,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/95cf21937e0a4d37b1f45fc7a44cdb56 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/95cf21937e0a4d37b1f45fc7a44cdb56 2024-11-24T08:15:30,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/95cf21937e0a4d37b1f45fc7a44cdb56, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T08:15:30,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/eef5c221e8b441fb9c2159834280af50 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/eef5c221e8b441fb9c2159834280af50 2024-11-24T08:15:30,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/eef5c221e8b441fb9c2159834280af50, entries=150, sequenceid=254, filesize=11.9 K 2024-11-24T08:15:30,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=46.96 KB/48090 for 36db097a55561465c9ef689006e62ac4 in 642ms, sequenceid=254, compaction requested=false 2024-11-24T08:15:30,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:30,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-24T08:15:30,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,891 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:15:30,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:30,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:30,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:30,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:30,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:30,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:30,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:30,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:30,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244c2711425efb44a6b331d8b7bb44259f_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436130246/Put/seqid=0 2024-11-24T08:15:30,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742199_1375 (size=12454) 2024-11-24T08:15:30,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:30,911 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244c2711425efb44a6b331d8b7bb44259f_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244c2711425efb44a6b331d8b7bb44259f_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:30,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/9fdad5d511bf4b6294321c32c82479b2, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:30,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/9fdad5d511bf4b6294321c32c82479b2 is 175, key is test_row_0/A:col10/1732436130246/Put/seqid=0 2024-11-24T08:15:30,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742200_1376 (size=31255) 2024-11-24T08:15:30,919 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/9fdad5d511bf4b6294321c32c82479b2 2024-11-24T08:15:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/72b7b88bca954c87868f961e28c0a9bb is 50, key is test_row_0/B:col10/1732436130246/Put/seqid=0 2024-11-24T08:15:30,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742201_1377 (size=12301) 2024-11-24T08:15:30,937 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/72b7b88bca954c87868f961e28c0a9bb 2024-11-24T08:15:30,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/911a0442e37f4a28a9ad588577008da7 is 50, key is test_row_0/C:col10/1732436130246/Put/seqid=0 2024-11-24T08:15:30,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742202_1378 (size=12301) 2024-11-24T08:15:30,955 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/911a0442e37f4a28a9ad588577008da7 2024-11-24T08:15:30,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/9fdad5d511bf4b6294321c32c82479b2 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/9fdad5d511bf4b6294321c32c82479b2 2024-11-24T08:15:30,963 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/9fdad5d511bf4b6294321c32c82479b2, entries=150, sequenceid=267, filesize=30.5 K 2024-11-24T08:15:30,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/72b7b88bca954c87868f961e28c0a9bb as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/72b7b88bca954c87868f961e28c0a9bb 2024-11-24T08:15:30,968 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/72b7b88bca954c87868f961e28c0a9bb, entries=150, sequenceid=267, filesize=12.0 K 2024-11-24T08:15:30,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/911a0442e37f4a28a9ad588577008da7 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/911a0442e37f4a28a9ad588577008da7 2024-11-24T08:15:30,972 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/911a0442e37f4a28a9ad588577008da7, entries=150, sequenceid=267, filesize=12.0 K 2024-11-24T08:15:30,975 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=107.34 KB/109920 for 36db097a55561465c9ef689006e62ac4 in 83ms, sequenceid=267, compaction requested=true 2024-11-24T08:15:30,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:30,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:30,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-24T08:15:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-24T08:15:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:30,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T08:15:30,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:30,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:30,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:30,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:30,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:30,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:30,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-24T08:15:30,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0780 sec 2024-11-24T08:15:30,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.0890 sec 2024-11-24T08:15:30,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124aebf769a4c3c4d458f6f12acaed67854_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436130952/Put/seqid=0 2024-11-24T08:15:30,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742203_1379 (size=14994) 2024-11-24T08:15:31,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436190991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436190994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436190995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436191004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-24T08:15:31,040 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:31,040 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-24T08:15:31,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T08:15:31,042 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:31,043 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:31,043 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:31,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436191109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436191109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436191120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436191120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T08:15:31,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T08:15:31,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:31,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:31,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:31,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436191330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436191330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T08:15:31,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:31,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:31,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:31,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436191340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436191340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T08:15:31,388 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:31,392 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124aebf769a4c3c4d458f6f12acaed67854_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124aebf769a4c3c4d458f6f12acaed67854_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:31,393 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/fa054ab1b8684015a1130e921d83f4da, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:31,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/fa054ab1b8684015a1130e921d83f4da is 175, key is test_row_0/A:col10/1732436130952/Put/seqid=0 2024-11-24T08:15:31,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742204_1380 (size=39949) 2024-11-24T08:15:31,398 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/fa054ab1b8684015a1130e921d83f4da 2024-11-24T08:15:31,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/d0cd615fd8714dcc9ee219a83de564cf is 50, key is test_row_0/B:col10/1732436130952/Put/seqid=0 2024-11-24T08:15:31,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742205_1381 (size=12301) 2024-11-24T08:15:31,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T08:15:31,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:31,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:31,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:31,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T08:15:31,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:31,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:31,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:31,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436191649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436191650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436191660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436191660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:31,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T08:15:31,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T08:15:31,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:31,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:31,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:31,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:31,814 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/d0cd615fd8714dcc9ee219a83de564cf 2024-11-24T08:15:31,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/a43918557e75465dbd8cadfe6bbfe4b5 is 50, key is test_row_0/C:col10/1732436130952/Put/seqid=0 2024-11-24T08:15:31,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742206_1382 (size=12301) 2024-11-24T08:15:31,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/a43918557e75465dbd8cadfe6bbfe4b5 2024-11-24T08:15:31,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/fa054ab1b8684015a1130e921d83f4da as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/fa054ab1b8684015a1130e921d83f4da 2024-11-24T08:15:31,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/fa054ab1b8684015a1130e921d83f4da, entries=200, sequenceid=287, filesize=39.0 K 2024-11-24T08:15:31,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/d0cd615fd8714dcc9ee219a83de564cf as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/d0cd615fd8714dcc9ee219a83de564cf 2024-11-24T08:15:31,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/d0cd615fd8714dcc9ee219a83de564cf, entries=150, sequenceid=287, filesize=12.0 K 2024-11-24T08:15:31,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/a43918557e75465dbd8cadfe6bbfe4b5 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/a43918557e75465dbd8cadfe6bbfe4b5 2024-11-24T08:15:31,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/a43918557e75465dbd8cadfe6bbfe4b5, entries=150, sequenceid=287, filesize=12.0 K 2024-11-24T08:15:31,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 36db097a55561465c9ef689006e62ac4 in 867ms, sequenceid=287, compaction requested=true 2024-11-24T08:15:31,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:31,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:31,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:31,843 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:15:31,843 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:15:31,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:31,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:31,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:31,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:31,844 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142298 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:15:31,844 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:15:31,844 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/B is initiating minor compaction (all files) 2024-11-24T08:15:31,844 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/A is initiating minor compaction (all files) 2024-11-24T08:15:31,844 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/B in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:31,844 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/A in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:31,844 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/6160b1dfa0074916968f37076b5ba386, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/c7b115a872e84373802d24fac659bf37, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/9fdad5d511bf4b6294321c32c82479b2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/fa054ab1b8684015a1130e921d83f4da] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=139.0 K 2024-11-24T08:15:31,844 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/302d4adff8844f5bb78351b37e4cd80c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/95cf21937e0a4d37b1f45fc7a44cdb56, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/72b7b88bca954c87868f961e28c0a9bb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/d0cd615fd8714dcc9ee219a83de564cf] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=48.0 K 2024-11-24T08:15:31,844 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:31,844 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/6160b1dfa0074916968f37076b5ba386, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/c7b115a872e84373802d24fac659bf37, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/9fdad5d511bf4b6294321c32c82479b2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/fa054ab1b8684015a1130e921d83f4da] 2024-11-24T08:15:31,844 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 302d4adff8844f5bb78351b37e4cd80c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732436126653 2024-11-24T08:15:31,844 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6160b1dfa0074916968f37076b5ba386, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732436126653 2024-11-24T08:15:31,845 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 95cf21937e0a4d37b1f45fc7a44cdb56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732436129023 2024-11-24T08:15:31,845 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7b115a872e84373802d24fac659bf37, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732436129012 2024-11-24T08:15:31,845 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b7b88bca954c87868f961e28c0a9bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732436130220 2024-11-24T08:15:31,845 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fdad5d511bf4b6294321c32c82479b2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732436130220 2024-11-24T08:15:31,845 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting d0cd615fd8714dcc9ee219a83de564cf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732436130952 2024-11-24T08:15:31,845 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa054ab1b8684015a1130e921d83f4da, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732436130952 2024-11-24T08:15:31,855 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:31,857 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#B#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:31,857 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/f57d68cfccf74eaa9394d39b4ab60e22 is 50, key is test_row_0/B:col10/1732436130952/Put/seqid=0 2024-11-24T08:15:31,861 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124b28edb348d0a4648af26697a605eb4df_36db097a55561465c9ef689006e62ac4 store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:31,863 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124b28edb348d0a4648af26697a605eb4df_36db097a55561465c9ef689006e62ac4, store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:31,864 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b28edb348d0a4648af26697a605eb4df_36db097a55561465c9ef689006e62ac4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:31,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742207_1383 (size=12677) 2024-11-24T08:15:31,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742208_1384 (size=4469) 2024-11-24T08:15:31,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-24T08:15:31,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:31,961 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T08:15:31,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:31,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:31,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:31,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:31,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:31,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:31,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249f6d08fe17eb411abae56e51f11707a6_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436131002/Put/seqid=0 2024-11-24T08:15:31,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742209_1385 (size=12454) 2024-11-24T08:15:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:32,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:32,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T08:15:32,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436192216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436192217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436192218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436192226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,279 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#A#compaction#322 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:32,279 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/249b491133084a3ea9aa8900c69100e1 is 175, key is test_row_0/A:col10/1732436130952/Put/seqid=0 2024-11-24T08:15:32,280 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/f57d68cfccf74eaa9394d39b4ab60e22 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/f57d68cfccf74eaa9394d39b4ab60e22 2024-11-24T08:15:32,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742210_1386 (size=31631) 2024-11-24T08:15:32,287 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36db097a55561465c9ef689006e62ac4/B of 36db097a55561465c9ef689006e62ac4 into f57d68cfccf74eaa9394d39b4ab60e22(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:32,287 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:32,287 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/B, priority=12, startTime=1732436131843; duration=0sec 2024-11-24T08:15:32,287 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:32,287 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:B 2024-11-24T08:15:32,287 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:15:32,291 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49144 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:15:32,291 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/C is initiating minor compaction (all files) 2024-11-24T08:15:32,291 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/C in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:32,291 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/c332eff5186842e5bc9c35a3ff88ab22, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/eef5c221e8b441fb9c2159834280af50, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/911a0442e37f4a28a9ad588577008da7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/a43918557e75465dbd8cadfe6bbfe4b5] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=48.0 K 2024-11-24T08:15:32,292 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting c332eff5186842e5bc9c35a3ff88ab22, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732436126653 2024-11-24T08:15:32,292 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/249b491133084a3ea9aa8900c69100e1 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/249b491133084a3ea9aa8900c69100e1 2024-11-24T08:15:32,293 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting eef5c221e8b441fb9c2159834280af50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1732436129023 2024-11-24T08:15:32,293 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 911a0442e37f4a28a9ad588577008da7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732436130220 2024-11-24T08:15:32,293 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting a43918557e75465dbd8cadfe6bbfe4b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732436130952 2024-11-24T08:15:32,299 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36db097a55561465c9ef689006e62ac4/A of 36db097a55561465c9ef689006e62ac4 into 249b491133084a3ea9aa8900c69100e1(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:32,299 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:32,300 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/A, priority=12, startTime=1732436131843; duration=0sec 2024-11-24T08:15:32,300 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:32,300 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:A 2024-11-24T08:15:32,308 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#C#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:32,309 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/051909a1acbe4c2e8c4c98418a7e99b4 is 50, key is test_row_0/C:col10/1732436130952/Put/seqid=0 2024-11-24T08:15:32,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742211_1387 (size=12677) 2024-11-24T08:15:32,318 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/051909a1acbe4c2e8c4c98418a7e99b4 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/051909a1acbe4c2e8c4c98418a7e99b4 2024-11-24T08:15:32,323 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36db097a55561465c9ef689006e62ac4/C of 36db097a55561465c9ef689006e62ac4 into 051909a1acbe4c2e8c4c98418a7e99b4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:32,323 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:32,323 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/C, priority=12, startTime=1732436131843; duration=0sec 2024-11-24T08:15:32,323 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:32,323 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:C 2024-11-24T08:15:32,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436192340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,354 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436192340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436192340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436192350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:32,378 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249f6d08fe17eb411abae56e51f11707a6_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249f6d08fe17eb411abae56e51f11707a6_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/57fe95a4ff374c588b17a6b4d0182edb, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/57fe95a4ff374c588b17a6b4d0182edb is 175, key is test_row_0/A:col10/1732436131002/Put/seqid=0 2024-11-24T08:15:32,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742212_1388 (size=31255) 2024-11-24T08:15:32,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436192560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436192560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436192560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436192560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,783 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/57fe95a4ff374c588b17a6b4d0182edb 2024-11-24T08:15:32,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/81aa91fa799a4a7aaff095813f23a1a4 is 50, key is test_row_0/B:col10/1732436131002/Put/seqid=0 2024-11-24T08:15:32,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742213_1389 (size=12301) 2024-11-24T08:15:32,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436192870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436192870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436192880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:32,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436192880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T08:15:33,194 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/81aa91fa799a4a7aaff095813f23a1a4 2024-11-24T08:15:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/a468575655254609b6ab3189abe7f4dc is 50, key is test_row_0/C:col10/1732436131002/Put/seqid=0 2024-11-24T08:15:33,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742214_1390 (size=12301) 2024-11-24T08:15:33,204 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/a468575655254609b6ab3189abe7f4dc 2024-11-24T08:15:33,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/57fe95a4ff374c588b17a6b4d0182edb as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/57fe95a4ff374c588b17a6b4d0182edb 2024-11-24T08:15:33,212 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/57fe95a4ff374c588b17a6b4d0182edb, entries=150, sequenceid=299, filesize=30.5 K 2024-11-24T08:15:33,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/81aa91fa799a4a7aaff095813f23a1a4 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/81aa91fa799a4a7aaff095813f23a1a4 2024-11-24T08:15:33,216 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/81aa91fa799a4a7aaff095813f23a1a4, entries=150, sequenceid=299, filesize=12.0 K 2024-11-24T08:15:33,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/a468575655254609b6ab3189abe7f4dc as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/a468575655254609b6ab3189abe7f4dc 2024-11-24T08:15:33,221 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/a468575655254609b6ab3189abe7f4dc, entries=150, sequenceid=299, filesize=12.0 K 2024-11-24T08:15:33,222 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 36db097a55561465c9ef689006e62ac4 in 1261ms, sequenceid=299, compaction requested=false 2024-11-24T08:15:33,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:33,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:33,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-24T08:15:33,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-24T08:15:33,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-24T08:15:33,224 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1800 sec 2024-11-24T08:15:33,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.1850 sec 2024-11-24T08:15:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:33,397 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-24T08:15:33,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:33,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:33,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:33,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:33,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:33,398 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:33,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244b39aa2a33dd443aaaaadd896d31f89f_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436132214/Put/seqid=0 2024-11-24T08:15:33,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742215_1391 (size=17534) 2024-11-24T08:15:33,409 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:33,413 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244b39aa2a33dd443aaaaadd896d31f89f_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244b39aa2a33dd443aaaaadd896d31f89f_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:33,414 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/55b5ccd63a364f89aab2b0e5c53064c2, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:33,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/55b5ccd63a364f89aab2b0e5c53064c2 is 175, key is test_row_0/A:col10/1732436132214/Put/seqid=0 2024-11-24T08:15:33,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742216_1392 (size=48639) 2024-11-24T08:15:33,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:33,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436193412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:33,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436193413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:33,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:33,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436193422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:33,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:33,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436193423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:33,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:33,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436193530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:33,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:33,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436193540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:33,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:33,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:33,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436193540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:33,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:33,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436193550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:33,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60138 deadline: 1732436193730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:33,744 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in 36db097a55561465c9ef689006e62ac4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:33,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:33,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436193740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:33,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436193760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:33,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436193760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:33,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436193770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:33,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:33,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:33,818 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/55b5ccd63a364f89aab2b0e5c53064c2 2024-11-24T08:15:33,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/0abb550c05ed48dda39ad18463bb6245 is 50, key is test_row_0/B:col10/1732436132214/Put/seqid=0 2024-11-24T08:15:33,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742217_1393 (size=12301) 2024-11-24T08:15:33,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/0abb550c05ed48dda39ad18463bb6245 2024-11-24T08:15:33,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/f8d957f7fb59419987b68817063df269 is 50, key is test_row_0/C:col10/1732436132214/Put/seqid=0 2024-11-24T08:15:33,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742218_1394 (size=12301) 2024-11-24T08:15:34,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436194060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:34,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:34,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:34,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436194080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:34,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:34,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436194080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:34,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:34,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436194089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:34,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,104 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:34,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/f8d957f7fb59419987b68817063df269 2024-11-24T08:15:34,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/55b5ccd63a364f89aab2b0e5c53064c2 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/55b5ccd63a364f89aab2b0e5c53064c2 2024-11-24T08:15:34,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/55b5ccd63a364f89aab2b0e5c53064c2, entries=250, sequenceid=324, filesize=47.5 K 2024-11-24T08:15:34,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/0abb550c05ed48dda39ad18463bb6245 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/0abb550c05ed48dda39ad18463bb6245 2024-11-24T08:15:34,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/0abb550c05ed48dda39ad18463bb6245, entries=150, sequenceid=324, filesize=12.0 K 2024-11-24T08:15:34,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/f8d957f7fb59419987b68817063df269 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/f8d957f7fb59419987b68817063df269 2024-11-24T08:15:34,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/f8d957f7fb59419987b68817063df269, entries=150, sequenceid=324, filesize=12.0 K 2024-11-24T08:15:34,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 36db097a55561465c9ef689006e62ac4 in 860ms, sequenceid=324, compaction requested=true 2024-11-24T08:15:34,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:34,257 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:34,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:34,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:34,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:34,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:34,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:34,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:34,257 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:34,258 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111525 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:34,258 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:34,258 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/B is initiating minor compaction (all files) 2024-11-24T08:15:34,258 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/A is initiating minor compaction (all files) 2024-11-24T08:15:34,258 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/B in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:34,258 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/A in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:34,259 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/f57d68cfccf74eaa9394d39b4ab60e22, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/81aa91fa799a4a7aaff095813f23a1a4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/0abb550c05ed48dda39ad18463bb6245] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=36.4 K 2024-11-24T08:15:34,259 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/249b491133084a3ea9aa8900c69100e1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/57fe95a4ff374c588b17a6b4d0182edb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/55b5ccd63a364f89aab2b0e5c53064c2] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=108.9 K 2024-11-24T08:15:34,259 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:34,259 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/249b491133084a3ea9aa8900c69100e1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/57fe95a4ff374c588b17a6b4d0182edb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/55b5ccd63a364f89aab2b0e5c53064c2] 2024-11-24T08:15:34,259 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting f57d68cfccf74eaa9394d39b4ab60e22, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732436130952 2024-11-24T08:15:34,259 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 249b491133084a3ea9aa8900c69100e1, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732436130952 2024-11-24T08:15:34,259 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 81aa91fa799a4a7aaff095813f23a1a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732436130990 2024-11-24T08:15:34,259 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57fe95a4ff374c588b17a6b4d0182edb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732436130990 2024-11-24T08:15:34,259 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0abb550c05ed48dda39ad18463bb6245, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732436132214 2024-11-24T08:15:34,260 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55b5ccd63a364f89aab2b0e5c53064c2, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732436132214 2024-11-24T08:15:34,266 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:34,266 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/ab043d39cf7746f198c5a553965ca985 is 50, key is test_row_0/B:col10/1732436132214/Put/seqid=0 2024-11-24T08:15:34,267 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:34,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742219_1395 (size=12779) 2024-11-24T08:15:34,273 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411248fae317de14443b38a57ffbfa055e4c9_36db097a55561465c9ef689006e62ac4 store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:34,275 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411248fae317de14443b38a57ffbfa055e4c9_36db097a55561465c9ef689006e62ac4, store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:34,275 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248fae317de14443b38a57ffbfa055e4c9_36db097a55561465c9ef689006e62ac4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:34,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742220_1396 (size=4469) 2024-11-24T08:15:34,284 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#A#compaction#331 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:34,285 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/4fa137f5e54c4925a2492e865701f0e9 is 175, key is test_row_0/A:col10/1732436132214/Put/seqid=0 2024-11-24T08:15:34,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742221_1397 (size=31733) 2024-11-24T08:15:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:34,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T08:15:34,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:34,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:34,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:34,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:34,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:34,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:34,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248e6490def62145fda08f8b618f8df1b9_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436133411/Put/seqid=0 2024-11-24T08:15:34,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742222_1398 (size=14994) 2024-11-24T08:15:34,675 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/ab043d39cf7746f198c5a553965ca985 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/ab043d39cf7746f198c5a553965ca985 2024-11-24T08:15:34,679 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36db097a55561465c9ef689006e62ac4/B of 36db097a55561465c9ef689006e62ac4 into ab043d39cf7746f198c5a553965ca985(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:34,679 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:34,679 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/B, priority=13, startTime=1732436134257; duration=0sec 2024-11-24T08:15:34,679 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:34,679 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:B 2024-11-24T08:15:34,679 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:34,680 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:34,680 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/C is initiating minor compaction (all files) 2024-11-24T08:15:34,680 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/C in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:34,680 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/051909a1acbe4c2e8c4c98418a7e99b4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/a468575655254609b6ab3189abe7f4dc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/f8d957f7fb59419987b68817063df269] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=36.4 K 2024-11-24T08:15:34,681 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 051909a1acbe4c2e8c4c98418a7e99b4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1732436130952 2024-11-24T08:15:34,681 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting a468575655254609b6ab3189abe7f4dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732436130990 2024-11-24T08:15:34,681 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting f8d957f7fb59419987b68817063df269, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732436132214 2024-11-24T08:15:34,687 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#C#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:34,687 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/07d248faf9984eaba4457245a92f1aef is 50, key is test_row_0/C:col10/1732436132214/Put/seqid=0 2024-11-24T08:15:34,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742223_1399 (size=12779) 2024-11-24T08:15:34,694 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/07d248faf9984eaba4457245a92f1aef as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/07d248faf9984eaba4457245a92f1aef 2024-11-24T08:15:34,696 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/4fa137f5e54c4925a2492e865701f0e9 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/4fa137f5e54c4925a2492e865701f0e9 2024-11-24T08:15:34,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436194696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:34,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1732436134655, max timestamp=1732436134695Num uniques 859; 2024-11-24T08:15:34,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T08:15:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:34,701 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36db097a55561465c9ef689006e62ac4/C of 36db097a55561465c9ef689006e62ac4 into 07d248faf9984eaba4457245a92f1aef(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:34,701 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:34,702 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/C, priority=13, startTime=1732436134257; duration=0sec 2024-11-24T08:15:34,702 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:34,702 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:C 2024-11-24T08:15:34,702 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36db097a55561465c9ef689006e62ac4/A of 36db097a55561465c9ef689006e62ac4 into 4fa137f5e54c4925a2492e865701f0e9(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:34,702 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:34,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=166, cellSize=7596, totalHeapSize=17504, min timestamp=1732436134656, max timestamp=1732436134695Num uniques 859; 2024-11-24T08:15:34,702 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/A, priority=13, startTime=1732436134257; duration=0sec 2024-11-24T08:15:34,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=7596, suffix heap size=117064,new segment heap size=17504 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=166 2024-11-24T08:15:34,702 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:34,702 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:A 2024-11-24T08:15:34,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732436134656, max timestamp=1732436134695Num uniques 859; 2024-11-24T08:15:34,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-24T08:15:34,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436194748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:34,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436194759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:34,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436194767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:34,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,774 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:34,798 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T08:15:34,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436194810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:34,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436194870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:34,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436194879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436194880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:35,001 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:35,005 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248e6490def62145fda08f8b618f8df1b9_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248e6490def62145fda08f8b618f8df1b9_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:35,006 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/8636219dcb394c8bb19e7335af6c9ed8, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:35,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/8636219dcb394c8bb19e7335af6c9ed8 is 175, key is test_row_0/A:col10/1732436133411/Put/seqid=0 2024-11-24T08:15:35,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742224_1400 (size=39949) 2024-11-24T08:15:35,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436195030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:35,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436195079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:35,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436195100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:35,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:35,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436195100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:35,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-24T08:15:35,200 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:35,200 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:35,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-24T08:15:35,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T08:15:35,202 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:35,203 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:35,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T08:15:35,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T08:15:35,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:35,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:35,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:35,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:35,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:35,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:35,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436195350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:35,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436195389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:35,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=335, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/8636219dcb394c8bb19e7335af6c9ed8 2024-11-24T08:15:35,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/25e8b213a36244c3bb454ebeeb98bc98 is 50, key is test_row_0/B:col10/1732436133411/Put/seqid=0 2024-11-24T08:15:35,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742225_1401 (size=12301) 2024-11-24T08:15:35,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436195420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:35,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436195420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:35,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T08:15:35,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:35,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:35,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:35,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:35,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:35,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T08:15:35,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T08:15:35,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:35,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:35,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:35,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:35,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:35,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:35,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T08:15:35,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:35,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:35,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:35,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:35,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:35,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/25e8b213a36244c3bb454ebeeb98bc98 2024-11-24T08:15:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T08:15:35,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/478b95545a50451193d543b9e249b1fa is 50, key is test_row_0/C:col10/1732436133411/Put/seqid=0 2024-11-24T08:15:35,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742226_1402 (size=12301) 2024-11-24T08:15:35,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:35,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436195869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:35,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:35,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:35,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436195910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:35,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:35,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436195930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:35,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436195930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:35,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:35,967 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T08:15:35,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:35,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:35,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:35,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:35,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:35,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:36,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T08:15:36,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:36,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:36,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:36,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:36,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:36,236 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/478b95545a50451193d543b9e249b1fa 2024-11-24T08:15:36,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/8636219dcb394c8bb19e7335af6c9ed8 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/8636219dcb394c8bb19e7335af6c9ed8 2024-11-24T08:15:36,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/8636219dcb394c8bb19e7335af6c9ed8, entries=200, sequenceid=335, filesize=39.0 K 2024-11-24T08:15:36,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/25e8b213a36244c3bb454ebeeb98bc98 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/25e8b213a36244c3bb454ebeeb98bc98 2024-11-24T08:15:36,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/25e8b213a36244c3bb454ebeeb98bc98, entries=150, sequenceid=335, filesize=12.0 K 2024-11-24T08:15:36,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/478b95545a50451193d543b9e249b1fa as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/478b95545a50451193d543b9e249b1fa 2024-11-24T08:15:36,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/478b95545a50451193d543b9e249b1fa, entries=150, sequenceid=335, filesize=12.0 K 2024-11-24T08:15:36,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.55 KB/128562 for 36db097a55561465c9ef689006e62ac4 in 1665ms, sequenceid=335, compaction requested=false 2024-11-24T08:15:36,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:36,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-24T08:15:36,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:36,273 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=125.55 KB heapSize=356.01 KB 2024-11-24T08:15:36,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:36,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:36,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:36,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:36,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:36,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:36,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249af62818a23d4860a57b84a1c43a79d2_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436134742/Put/seqid=0 2024-11-24T08:15:36,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742227_1403 (size=12454) 2024-11-24T08:15:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T08:15:36,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:36,690 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411249af62818a23d4860a57b84a1c43a79d2_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249af62818a23d4860a57b84a1c43a79d2_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:36,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/db2b14148bf9472f87ed64260068a8e6, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:36,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/db2b14148bf9472f87ed64260068a8e6 is 175, key is test_row_0/A:col10/1732436134742/Put/seqid=0 2024-11-24T08:15:36,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742228_1404 (size=31255) 2024-11-24T08:15:36,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:36,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:36,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:36,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436196934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:36,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:36,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436196940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:36,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:36,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:36,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436196949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:36,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:36,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436196949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:36,956 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in 36db097a55561465c9ef689006e62ac4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:36,957 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 36db097a55561465c9ef689006e62ac4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:37,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436197050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:37,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:37,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:37,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436197059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:37,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:37,101 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/db2b14148bf9472f87ed64260068a8e6 2024-11-24T08:15:37,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/60d3bcea687c4b2e82ba48f71530fe57 is 50, key is test_row_0/B:col10/1732436134742/Put/seqid=0 2024-11-24T08:15:37,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742229_1405 (size=12301) 2024-11-24T08:15:37,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:37,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436197270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:37,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:37,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436197280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:37,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:37,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T08:15:37,514 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.57 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/60d3bcea687c4b2e82ba48f71530fe57 2024-11-24T08:15:37,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/77e294b099c047dcbf37edee62f07cb1 is 50, key is test_row_0/C:col10/1732436134742/Put/seqid=0 2024-11-24T08:15:37,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742230_1406 (size=12301) 2024-11-24T08:15:37,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:37,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436197590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:37,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:37,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:37,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436197600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:37,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:37,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:37,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:37,925 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/77e294b099c047dcbf37edee62f07cb1 2024-11-24T08:15:37,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/db2b14148bf9472f87ed64260068a8e6 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/db2b14148bf9472f87ed64260068a8e6 2024-11-24T08:15:37,933 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/db2b14148bf9472f87ed64260068a8e6, entries=150, sequenceid=373, filesize=30.5 K 2024-11-24T08:15:37,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/60d3bcea687c4b2e82ba48f71530fe57 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/60d3bcea687c4b2e82ba48f71530fe57 2024-11-24T08:15:37,936 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/60d3bcea687c4b2e82ba48f71530fe57, entries=150, sequenceid=373, filesize=12.0 K 2024-11-24T08:15:37,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/77e294b099c047dcbf37edee62f07cb1 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/77e294b099c047dcbf37edee62f07cb1 2024-11-24T08:15:37,940 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/77e294b099c047dcbf37edee62f07cb1, entries=150, sequenceid=373, filesize=12.0 K 2024-11-24T08:15:37,941 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~125.55 KB/128562, heapSize ~355.96 KB/364504, currentSize=53.67 KB/54960 for 36db097a55561465c9ef689006e62ac4 in 1668ms, sequenceid=373, compaction requested=true 2024-11-24T08:15:37,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:37,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:37,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-24T08:15:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-24T08:15:37,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-24T08:15:37,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7390 sec 2024-11-24T08:15:37,945 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.7430 sec 2024-11-24T08:15:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:38,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T08:15:38,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:38,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:38,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:38,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:38,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:38,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:38,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240824f65ba0fb4661816bd3087db4ad93_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436138113/Put/seqid=0 2024-11-24T08:15:38,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742231_1407 (size=14994) 2024-11-24T08:15:38,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436198201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:38,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436198206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:38,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,217 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:38,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436198319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:38,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436198319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:38,491 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/6505cb74fead408298f4e30ed5a9e9c4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/55de1a9d081d4ccea9c4126d530a930b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/ad07df2b09e144f8935a6e769642bcb0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/0ff347a4da4e4778a2d78f196dc8c454, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/e15f7fe39f8c4c18a92f71f9022537bc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/c0472b0da4984ddbb5934045e05a8db6, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/21667912d2ce4e0488bd5546cf17cefd, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/6160b1dfa0074916968f37076b5ba386, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/c7b115a872e84373802d24fac659bf37, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/9fdad5d511bf4b6294321c32c82479b2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/fa054ab1b8684015a1130e921d83f4da, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/249b491133084a3ea9aa8900c69100e1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/57fe95a4ff374c588b17a6b4d0182edb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/55b5ccd63a364f89aab2b0e5c53064c2] to archive 2024-11-24T08:15:38,492 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:15:38,493 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/6505cb74fead408298f4e30ed5a9e9c4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/6505cb74fead408298f4e30ed5a9e9c4 2024-11-24T08:15:38,494 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/55de1a9d081d4ccea9c4126d530a930b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/55de1a9d081d4ccea9c4126d530a930b 2024-11-24T08:15:38,495 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/ad07df2b09e144f8935a6e769642bcb0 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/ad07df2b09e144f8935a6e769642bcb0 2024-11-24T08:15:38,496 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/0ff347a4da4e4778a2d78f196dc8c454 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/0ff347a4da4e4778a2d78f196dc8c454 2024-11-24T08:15:38,497 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/e15f7fe39f8c4c18a92f71f9022537bc to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/e15f7fe39f8c4c18a92f71f9022537bc 2024-11-24T08:15:38,498 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/c0472b0da4984ddbb5934045e05a8db6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/c0472b0da4984ddbb5934045e05a8db6 2024-11-24T08:15:38,499 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/21667912d2ce4e0488bd5546cf17cefd to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/21667912d2ce4e0488bd5546cf17cefd 2024-11-24T08:15:38,500 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/6160b1dfa0074916968f37076b5ba386 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/6160b1dfa0074916968f37076b5ba386 2024-11-24T08:15:38,501 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/c7b115a872e84373802d24fac659bf37 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/c7b115a872e84373802d24fac659bf37 2024-11-24T08:15:38,502 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/9fdad5d511bf4b6294321c32c82479b2 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/9fdad5d511bf4b6294321c32c82479b2 2024-11-24T08:15:38,503 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/fa054ab1b8684015a1130e921d83f4da to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/fa054ab1b8684015a1130e921d83f4da 2024-11-24T08:15:38,505 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/249b491133084a3ea9aa8900c69100e1 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/249b491133084a3ea9aa8900c69100e1 2024-11-24T08:15:38,505 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/57fe95a4ff374c588b17a6b4d0182edb to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/57fe95a4ff374c588b17a6b4d0182edb 2024-11-24T08:15:38,506 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/55b5ccd63a364f89aab2b0e5c53064c2 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/55b5ccd63a364f89aab2b0e5c53064c2 2024-11-24T08:15:38,509 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/758c2f05060843b5acf8a892ed9d365d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/c93a8babb07f4a0db5aeb21ea96d8a6d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/88346d2a7c754d32ba570b6d1f246ade, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/83e611a8b8a941e7a0dc2c2fc9f91198, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/4917a478031d44709b0e774d5ea3bc7e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/4ecca390174a4cc4a1eace1c068f8c5f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/302d4adff8844f5bb78351b37e4cd80c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/8a09c718601c42abbdcb3683d8fdf898, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/95cf21937e0a4d37b1f45fc7a44cdb56, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/72b7b88bca954c87868f961e28c0a9bb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/f57d68cfccf74eaa9394d39b4ab60e22, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/d0cd615fd8714dcc9ee219a83de564cf, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/81aa91fa799a4a7aaff095813f23a1a4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/0abb550c05ed48dda39ad18463bb6245] to archive 2024-11-24T08:15:38,510 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:15:38,511 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/758c2f05060843b5acf8a892ed9d365d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/758c2f05060843b5acf8a892ed9d365d 2024-11-24T08:15:38,512 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/c93a8babb07f4a0db5aeb21ea96d8a6d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/c93a8babb07f4a0db5aeb21ea96d8a6d 2024-11-24T08:15:38,513 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/88346d2a7c754d32ba570b6d1f246ade to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/88346d2a7c754d32ba570b6d1f246ade 2024-11-24T08:15:38,514 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/83e611a8b8a941e7a0dc2c2fc9f91198 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/83e611a8b8a941e7a0dc2c2fc9f91198 2024-11-24T08:15:38,515 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/4917a478031d44709b0e774d5ea3bc7e to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/4917a478031d44709b0e774d5ea3bc7e 2024-11-24T08:15:38,516 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/4ecca390174a4cc4a1eace1c068f8c5f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/4ecca390174a4cc4a1eace1c068f8c5f 2024-11-24T08:15:38,518 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/302d4adff8844f5bb78351b37e4cd80c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/302d4adff8844f5bb78351b37e4cd80c 2024-11-24T08:15:38,519 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/8a09c718601c42abbdcb3683d8fdf898 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/8a09c718601c42abbdcb3683d8fdf898 2024-11-24T08:15:38,520 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/95cf21937e0a4d37b1f45fc7a44cdb56 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/95cf21937e0a4d37b1f45fc7a44cdb56 2024-11-24T08:15:38,521 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/72b7b88bca954c87868f961e28c0a9bb to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/72b7b88bca954c87868f961e28c0a9bb 2024-11-24T08:15:38,522 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/f57d68cfccf74eaa9394d39b4ab60e22 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/f57d68cfccf74eaa9394d39b4ab60e22 2024-11-24T08:15:38,523 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/d0cd615fd8714dcc9ee219a83de564cf to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/d0cd615fd8714dcc9ee219a83de564cf 2024-11-24T08:15:38,524 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/81aa91fa799a4a7aaff095813f23a1a4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/81aa91fa799a4a7aaff095813f23a1a4 2024-11-24T08:15:38,526 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:38,526 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/0abb550c05ed48dda39ad18463bb6245 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/0abb550c05ed48dda39ad18463bb6245 2024-11-24T08:15:38,529 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/fff4b2bd0cf64988a7b12b6559a7eb56, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/b7a99bd56ea247d8915beafc672b5806, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/79e8433404c5421e9ddf114983ed173d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/13ffa5a706d646149ae70d0e1114d83b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/05e184e20746461b92e6a5aef03eef00, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/9a313c18899b4cf0928ac19d26cb6a22, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/c332eff5186842e5bc9c35a3ff88ab22, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/b706db157c284c6283ab8d2c06d240e5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/eef5c221e8b441fb9c2159834280af50, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/911a0442e37f4a28a9ad588577008da7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/051909a1acbe4c2e8c4c98418a7e99b4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/a43918557e75465dbd8cadfe6bbfe4b5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/a468575655254609b6ab3189abe7f4dc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/f8d957f7fb59419987b68817063df269] to archive 2024-11-24T08:15:38,529 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240824f65ba0fb4661816bd3087db4ad93_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240824f65ba0fb4661816bd3087db4ad93_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:38,530 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:15:38,530 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/dde716388f3a4eedbb2e52aecfa0f10f, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:38,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/dde716388f3a4eedbb2e52aecfa0f10f is 175, key is test_row_0/A:col10/1732436138113/Put/seqid=0 2024-11-24T08:15:38,531 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/fff4b2bd0cf64988a7b12b6559a7eb56 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/fff4b2bd0cf64988a7b12b6559a7eb56 2024-11-24T08:15:38,532 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/b7a99bd56ea247d8915beafc672b5806 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/b7a99bd56ea247d8915beafc672b5806 2024-11-24T08:15:38,533 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/79e8433404c5421e9ddf114983ed173d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/79e8433404c5421e9ddf114983ed173d 2024-11-24T08:15:38,534 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/13ffa5a706d646149ae70d0e1114d83b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/13ffa5a706d646149ae70d0e1114d83b 2024-11-24T08:15:38,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742232_1408 (size=39949) 2024-11-24T08:15:38,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:38,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436198529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:38,537 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/05e184e20746461b92e6a5aef03eef00 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/05e184e20746461b92e6a5aef03eef00 2024-11-24T08:15:38,538 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/9a313c18899b4cf0928ac19d26cb6a22 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/9a313c18899b4cf0928ac19d26cb6a22 2024-11-24T08:15:38,539 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/c332eff5186842e5bc9c35a3ff88ab22 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/c332eff5186842e5bc9c35a3ff88ab22 2024-11-24T08:15:38,541 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/b706db157c284c6283ab8d2c06d240e5 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/b706db157c284c6283ab8d2c06d240e5 2024-11-24T08:15:38,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/eef5c221e8b441fb9c2159834280af50 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/eef5c221e8b441fb9c2159834280af50 2024-11-24T08:15:38,542 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/911a0442e37f4a28a9ad588577008da7 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/911a0442e37f4a28a9ad588577008da7 2024-11-24T08:15:38,543 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/051909a1acbe4c2e8c4c98418a7e99b4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/051909a1acbe4c2e8c4c98418a7e99b4 2024-11-24T08:15:38,544 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/a43918557e75465dbd8cadfe6bbfe4b5 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/a43918557e75465dbd8cadfe6bbfe4b5 2024-11-24T08:15:38,545 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/a468575655254609b6ab3189abe7f4dc to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/a468575655254609b6ab3189abe7f4dc 2024-11-24T08:15:38,546 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/f8d957f7fb59419987b68817063df269 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/f8d957f7fb59419987b68817063df269 2024-11-24T08:15:38,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436198540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,553 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:38,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436198840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:38,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436198860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:38,936 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=385, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/dde716388f3a4eedbb2e52aecfa0f10f 2024-11-24T08:15:38,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/80c63b6197be45db8a66fd2734981f92 is 50, key is test_row_0/B:col10/1732436138113/Put/seqid=0 2024-11-24T08:15:38,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742233_1409 (size=12301) 2024-11-24T08:15:38,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60198 deadline: 1732436198970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:38,980 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in 36db097a55561465c9ef689006e62ac4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:38,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:38,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60160 deadline: 1732436198980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:38,988 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in 36db097a55561465c9ef689006e62ac4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:39,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/80c63b6197be45db8a66fd2734981f92 2024-11-24T08:15:39,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/99b5077a927d4b91bd0e556b158f3cec is 50, key is test_row_0/C:col10/1732436138113/Put/seqid=0 2024-11-24T08:15:39,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742234_1410 (size=12301) 2024-11-24T08:15:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-24T08:15:39,360 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:39,361 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-24T08:15:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T08:15:39,363 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:39,363 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:39,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:39,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436199360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:39,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436199379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T08:15:39,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T08:15:39,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:39,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:39,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:39,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:39,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:39,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:39,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T08:15:39,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:39,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:39,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:39,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:39,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:39,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T08:15:39,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/99b5077a927d4b91bd0e556b158f3cec 2024-11-24T08:15:39,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/dde716388f3a4eedbb2e52aecfa0f10f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/dde716388f3a4eedbb2e52aecfa0f10f 2024-11-24T08:15:39,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/dde716388f3a4eedbb2e52aecfa0f10f, entries=200, sequenceid=385, filesize=39.0 K 2024-11-24T08:15:39,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/80c63b6197be45db8a66fd2734981f92 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/80c63b6197be45db8a66fd2734981f92 2024-11-24T08:15:39,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/80c63b6197be45db8a66fd2734981f92, entries=150, sequenceid=385, filesize=12.0 K 2024-11-24T08:15:39,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/99b5077a927d4b91bd0e556b158f3cec as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/99b5077a927d4b91bd0e556b158f3cec 2024-11-24T08:15:39,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/99b5077a927d4b91bd0e556b158f3cec, entries=150, sequenceid=385, filesize=12.0 K 2024-11-24T08:15:39,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 36db097a55561465c9ef689006e62ac4 in 1658ms, sequenceid=385, compaction requested=true 2024-11-24T08:15:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:39,773 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:15:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:39,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:39,773 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:15:39,774 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:15:39,774 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:15:39,774 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/A is initiating minor compaction (all files) 2024-11-24T08:15:39,774 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/B is initiating minor compaction (all files) 2024-11-24T08:15:39,774 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/A in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:39,774 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/B in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:39,774 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/4fa137f5e54c4925a2492e865701f0e9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/8636219dcb394c8bb19e7335af6c9ed8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/db2b14148bf9472f87ed64260068a8e6, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/dde716388f3a4eedbb2e52aecfa0f10f] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=139.5 K 2024-11-24T08:15:39,774 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/ab043d39cf7746f198c5a553965ca985, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/25e8b213a36244c3bb454ebeeb98bc98, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/60d3bcea687c4b2e82ba48f71530fe57, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/80c63b6197be45db8a66fd2734981f92] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=48.5 K 2024-11-24T08:15:39,774 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:39,774 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/4fa137f5e54c4925a2492e865701f0e9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/8636219dcb394c8bb19e7335af6c9ed8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/db2b14148bf9472f87ed64260068a8e6, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/dde716388f3a4eedbb2e52aecfa0f10f] 2024-11-24T08:15:39,774 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting ab043d39cf7746f198c5a553965ca985, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732436132214 2024-11-24T08:15:39,775 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fa137f5e54c4925a2492e865701f0e9, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732436132214 2024-11-24T08:15:39,775 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 25e8b213a36244c3bb454ebeeb98bc98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732436133411 2024-11-24T08:15:39,775 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8636219dcb394c8bb19e7335af6c9ed8, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732436133411 2024-11-24T08:15:39,775 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 60d3bcea687c4b2e82ba48f71530fe57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732436134740 2024-11-24T08:15:39,775 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting db2b14148bf9472f87ed64260068a8e6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732436134740 2024-11-24T08:15:39,775 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 80c63b6197be45db8a66fd2734981f92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732436136924 2024-11-24T08:15:39,775 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting dde716388f3a4eedbb2e52aecfa0f10f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732436136924 2024-11-24T08:15:39,782 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:39,783 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411248165beac0c8d479886a65d3ecbb42974_36db097a55561465c9ef689006e62ac4 store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:39,784 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#B#compaction#343 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:39,784 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/f1af1126a0414b23b622aba73a86f42a is 50, key is test_row_0/B:col10/1732436138113/Put/seqid=0 2024-11-24T08:15:39,786 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411248165beac0c8d479886a65d3ecbb42974_36db097a55561465c9ef689006e62ac4, store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:39,786 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411248165beac0c8d479886a65d3ecbb42974_36db097a55561465c9ef689006e62ac4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:39,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742236_1412 (size=4469) 2024-11-24T08:15:39,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742235_1411 (size=12439) 2024-11-24T08:15:39,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-24T08:15:39,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:39,822 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T08:15:39,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:39,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:39,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:39,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:39,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:39,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:39,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124052a9e71d2654c7488d44d06fef581f2_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436138204/Put/seqid=0 2024-11-24T08:15:39,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742237_1413 (size=12454) 2024-11-24T08:15:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T08:15:40,199 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#A#compaction#342 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:40,200 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/5cf33751e17c48caa93540324f4b603e is 175, key is test_row_0/A:col10/1732436138113/Put/seqid=0 2024-11-24T08:15:40,203 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/f1af1126a0414b23b622aba73a86f42a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/f1af1126a0414b23b622aba73a86f42a 2024-11-24T08:15:40,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742238_1414 (size=31393) 2024-11-24T08:15:40,208 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36db097a55561465c9ef689006e62ac4/B of 36db097a55561465c9ef689006e62ac4 into f1af1126a0414b23b622aba73a86f42a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:40,208 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:40,209 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/B, priority=12, startTime=1732436139773; duration=0sec 2024-11-24T08:15:40,209 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:40,209 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:B 2024-11-24T08:15:40,209 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:15:40,209 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/5cf33751e17c48caa93540324f4b603e as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/5cf33751e17c48caa93540324f4b603e 2024-11-24T08:15:40,210 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:15:40,210 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/C is initiating minor compaction (all files) 2024-11-24T08:15:40,210 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/C in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:40,210 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/07d248faf9984eaba4457245a92f1aef, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/478b95545a50451193d543b9e249b1fa, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/77e294b099c047dcbf37edee62f07cb1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/99b5077a927d4b91bd0e556b158f3cec] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=48.5 K 2024-11-24T08:15:40,211 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 07d248faf9984eaba4457245a92f1aef, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732436132214 2024-11-24T08:15:40,211 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 478b95545a50451193d543b9e249b1fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1732436133411 2024-11-24T08:15:40,211 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 77e294b099c047dcbf37edee62f07cb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732436134740 2024-11-24T08:15:40,212 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 99b5077a927d4b91bd0e556b158f3cec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732436136924 2024-11-24T08:15:40,214 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36db097a55561465c9ef689006e62ac4/A of 36db097a55561465c9ef689006e62ac4 into 5cf33751e17c48caa93540324f4b603e(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:40,214 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:40,214 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/A, priority=12, startTime=1732436139773; duration=0sec 2024-11-24T08:15:40,214 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:40,214 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:A 2024-11-24T08:15:40,219 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#C#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:40,220 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/6eedbee977404f479f5dc3e159015d60 is 50, key is test_row_0/C:col10/1732436138113/Put/seqid=0 2024-11-24T08:15:40,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742239_1415 (size=12439) 2024-11-24T08:15:40,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:40,237 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124052a9e71d2654c7488d44d06fef581f2_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124052a9e71d2654c7488d44d06fef581f2_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:40,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/267a2daefcc54ceba61dabba8ca77285, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:40,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/267a2daefcc54ceba61dabba8ca77285 is 175, key is test_row_0/A:col10/1732436138204/Put/seqid=0 2024-11-24T08:15:40,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742240_1416 (size=31255) 2024-11-24T08:15:40,245 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=405, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/267a2daefcc54ceba61dabba8ca77285 2024-11-24T08:15:40,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/ddd3b3fbe10144ee859681c440086b18 is 50, key is test_row_0/B:col10/1732436138204/Put/seqid=0 2024-11-24T08:15:40,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742241_1417 (size=12301) 2024-11-24T08:15:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:40,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. as already flushing 2024-11-24T08:15:40,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436200424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:40,434 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:40,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:40,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436200434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:40,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T08:15:40,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436200540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:40,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:40,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:40,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436200540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:40,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:40,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:40,570 INFO [Thread-1558 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1558. 2024-11-24T08:15:40,570 DEBUG [Thread-1558 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:40,570 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:40,570 DEBUG [Thread-1558 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:40,570 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:40,574 INFO [Thread-1560 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1560. 2024-11-24T08:15:40,574 DEBUG [Thread-1560 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:40,574 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:40,574 DEBUG [Thread-1560 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:40,574 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:40,578 INFO [Thread-1552 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1552. 2024-11-24T08:15:40,578 DEBUG [Thread-1552 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:40,578 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:40,578 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:40,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:40,581 INFO [Thread-1554 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1554. 2024-11-24T08:15:40,581 DEBUG [Thread-1554 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:40,581 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:40,581 DEBUG [Thread-1554 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:40,581 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:40,583 INFO [Thread-1556 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1556. 2024-11-24T08:15:40,583 DEBUG [Thread-1556 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:40,583 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:40,583 DEBUG [Thread-1556 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:40,583 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:40,627 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/6eedbee977404f479f5dc3e159015d60 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/6eedbee977404f479f5dc3e159015d60 2024-11-24T08:15:40,630 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 36db097a55561465c9ef689006e62ac4/C of 36db097a55561465c9ef689006e62ac4 into 6eedbee977404f479f5dc3e159015d60(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:40,631 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:40,631 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/C, priority=12, startTime=1732436139773; duration=0sec 2024-11-24T08:15:40,631 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:40,631 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:C 2024-11-24T08:15:40,660 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/ddd3b3fbe10144ee859681c440086b18 2024-11-24T08:15:40,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/fde1103e726748f9ba857ea90111b12e is 50, key is test_row_0/C:col10/1732436138204/Put/seqid=0 2024-11-24T08:15:40,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742242_1418 (size=12301) 2024-11-24T08:15:40,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:40,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436200749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436200749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:40,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:41,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:41,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60158 deadline: 1732436201060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60208 deadline: 1732436201060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:41,061 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:15:41,069 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/fde1103e726748f9ba857ea90111b12e 2024-11-24T08:15:41,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/267a2daefcc54ceba61dabba8ca77285 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/267a2daefcc54ceba61dabba8ca77285 2024-11-24T08:15:41,075 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/267a2daefcc54ceba61dabba8ca77285, entries=150, sequenceid=405, filesize=30.5 K 2024-11-24T08:15:41,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/ddd3b3fbe10144ee859681c440086b18 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/ddd3b3fbe10144ee859681c440086b18 2024-11-24T08:15:41,078 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/ddd3b3fbe10144ee859681c440086b18, entries=150, sequenceid=405, filesize=12.0 K 2024-11-24T08:15:41,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/fde1103e726748f9ba857ea90111b12e as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/fde1103e726748f9ba857ea90111b12e 2024-11-24T08:15:41,081 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/fde1103e726748f9ba857ea90111b12e, entries=150, sequenceid=405, filesize=12.0 K 2024-11-24T08:15:41,082 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 36db097a55561465c9ef689006e62ac4 in 1260ms, sequenceid=405, compaction requested=false 2024-11-24T08:15:41,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:41,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:41,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-24T08:15:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-24T08:15:41,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-24T08:15:41,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7200 sec 2024-11-24T08:15:41,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.7240 sec 2024-11-24T08:15:41,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-24T08:15:41,510 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:41,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:41,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T08:15:41,571 INFO [Thread-1545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1545. 2024-11-24T08:15:41,571 INFO [Thread-1541 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1541. 2024-11-24T08:15:41,571 DEBUG [Thread-1545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:41,571 DEBUG [Thread-1541 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:41,571 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:41,571 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:41,571 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:41,571 DEBUG [Thread-1545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:41,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:41,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:41,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:41,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240c218df0a27541f39559f4f302f9fb9c_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_0/A:col10/1732436140422/Put/seqid=0 2024-11-24T08:15:41,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742243_1419 (size=12454) 2024-11-24T08:15:41,981 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:41,984 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411240c218df0a27541f39559f4f302f9fb9c_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240c218df0a27541f39559f4f302f9fb9c_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:41,984 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/24d1f3d95428463ba7e27bbdb89c0a13, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:41,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/24d1f3d95428463ba7e27bbdb89c0a13 is 175, key is test_row_0/A:col10/1732436140422/Put/seqid=0 2024-11-24T08:15:41,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742244_1420 (size=31255) 2024-11-24T08:15:42,388 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=422, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/24d1f3d95428463ba7e27bbdb89c0a13 2024-11-24T08:15:42,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/57507be8b5c343038a829770ef38f34b is 50, key is test_row_0/B:col10/1732436140422/Put/seqid=0 2024-11-24T08:15:42,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742245_1421 (size=12301) 2024-11-24T08:15:42,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/57507be8b5c343038a829770ef38f34b 2024-11-24T08:15:42,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/4b30384fc93e44e184147286f9c08406 is 50, key is test_row_0/C:col10/1732436140422/Put/seqid=0 2024-11-24T08:15:42,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742246_1422 (size=12301) 2024-11-24T08:15:43,032 INFO [Thread-1549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1549. 2024-11-24T08:15:43,032 INFO [Thread-1543 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1543. 2024-11-24T08:15:43,032 DEBUG [Thread-1543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:43,032 DEBUG [Thread-1549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:43,032 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:43,032 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:43,032 DEBUG [Thread-1543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:43,032 DEBUG [Thread-1549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:43,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:43,032 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:43,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=422 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/4b30384fc93e44e184147286f9c08406 2024-11-24T08:15:43,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/24d1f3d95428463ba7e27bbdb89c0a13 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/24d1f3d95428463ba7e27bbdb89c0a13 2024-11-24T08:15:43,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/24d1f3d95428463ba7e27bbdb89c0a13, entries=150, sequenceid=422, filesize=30.5 K 2024-11-24T08:15:43,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/57507be8b5c343038a829770ef38f34b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/57507be8b5c343038a829770ef38f34b 2024-11-24T08:15:43,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/57507be8b5c343038a829770ef38f34b, entries=150, sequenceid=422, filesize=12.0 K 2024-11-24T08:15:43,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/4b30384fc93e44e184147286f9c08406 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/4b30384fc93e44e184147286f9c08406 2024-11-24T08:15:43,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/4b30384fc93e44e184147286f9c08406, entries=150, sequenceid=422, filesize=12.0 K 2024-11-24T08:15:43,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=13.42 KB/13740 for 36db097a55561465c9ef689006e62ac4 in 1649ms, sequenceid=422, compaction requested=true 2024-11-24T08:15:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:43,221 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 36db097a55561465c9ef689006e62ac4:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:43,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:43,221 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:43,221 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:43,221 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37041 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:43,222 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/A is initiating minor compaction (all files) 2024-11-24T08:15:43,222 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/B is initiating minor compaction (all files) 2024-11-24T08:15:43,222 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/A in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:43,222 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/B in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:43,222 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/5cf33751e17c48caa93540324f4b603e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/267a2daefcc54ceba61dabba8ca77285, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/24d1f3d95428463ba7e27bbdb89c0a13] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=91.7 K 2024-11-24T08:15:43,222 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/f1af1126a0414b23b622aba73a86f42a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/ddd3b3fbe10144ee859681c440086b18, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/57507be8b5c343038a829770ef38f34b] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=36.2 K 2024-11-24T08:15:43,222 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:43,222 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/5cf33751e17c48caa93540324f4b603e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/267a2daefcc54ceba61dabba8ca77285, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/24d1f3d95428463ba7e27bbdb89c0a13] 2024-11-24T08:15:43,222 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting f1af1126a0414b23b622aba73a86f42a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732436136924 2024-11-24T08:15:43,222 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cf33751e17c48caa93540324f4b603e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732436136924 2024-11-24T08:15:43,222 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting ddd3b3fbe10144ee859681c440086b18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732436138173 2024-11-24T08:15:43,222 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 267a2daefcc54ceba61dabba8ca77285, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732436138173 2024-11-24T08:15:43,223 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 57507be8b5c343038a829770ef38f34b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732436140422 2024-11-24T08:15:43,223 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24d1f3d95428463ba7e27bbdb89c0a13, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732436140422 2024-11-24T08:15:43,229 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#B#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:43,229 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/c345dadb4c1b4ec2ad1a49efec4c699d is 50, key is test_row_0/B:col10/1732436140422/Put/seqid=0 2024-11-24T08:15:43,232 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:43,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742247_1423 (size=12541) 2024-11-24T08:15:43,234 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124e86beb414c124934bc6852f03f60ec56_36db097a55561465c9ef689006e62ac4 store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:43,247 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124e86beb414c124934bc6852f03f60ec56_36db097a55561465c9ef689006e62ac4, store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:43,247 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e86beb414c124934bc6852f03f60ec56_36db097a55561465c9ef689006e62ac4 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:43,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742248_1424 (size=4469) 2024-11-24T08:15:43,637 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/c345dadb4c1b4ec2ad1a49efec4c699d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/c345dadb4c1b4ec2ad1a49efec4c699d 2024-11-24T08:15:43,641 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36db097a55561465c9ef689006e62ac4/B of 36db097a55561465c9ef689006e62ac4 into c345dadb4c1b4ec2ad1a49efec4c699d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:43,641 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:43,641 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/B, priority=13, startTime=1732436143221; duration=0sec 2024-11-24T08:15:43,641 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:43,641 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:B 2024-11-24T08:15:43,641 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:43,642 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37041 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:43,642 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): 36db097a55561465c9ef689006e62ac4/C is initiating minor compaction (all files) 2024-11-24T08:15:43,642 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 36db097a55561465c9ef689006e62ac4/C in TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:43,642 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/6eedbee977404f479f5dc3e159015d60, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/fde1103e726748f9ba857ea90111b12e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/4b30384fc93e44e184147286f9c08406] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp, totalSize=36.2 K 2024-11-24T08:15:43,642 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eedbee977404f479f5dc3e159015d60, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732436136924 2024-11-24T08:15:43,642 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting fde1103e726748f9ba857ea90111b12e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1732436138173 2024-11-24T08:15:43,643 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b30384fc93e44e184147286f9c08406, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=422, earliestPutTs=1732436140422 2024-11-24T08:15:43,649 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:43,650 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/86c06d2de8df4e0eb87cb9f063c2a294 is 50, key is test_row_0/C:col10/1732436140422/Put/seqid=0 2024-11-24T08:15:43,652 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 36db097a55561465c9ef689006e62ac4#A#compaction#352 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:43,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742249_1425 (size=12541) 2024-11-24T08:15:43,653 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/061a95ee25644d128348732e427c9893 is 175, key is test_row_0/A:col10/1732436140422/Put/seqid=0 2024-11-24T08:15:43,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742250_1426 (size=31495) 2024-11-24T08:15:43,841 INFO [Thread-1547 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1547. 2024-11-24T08:15:43,841 DEBUG [Thread-1547 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:15:43,841 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:43,842 DEBUG [Thread-1547 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:43,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 906 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2713 rows 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 900 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2698 rows 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 902 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2702 rows 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 901 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2698 rows 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 908 2024-11-24T08:15:43,842 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2718 rows 2024-11-24T08:15:43,842 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T08:15:43,842 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:15:43,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:43,842 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:43,842 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:15:43,843 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T08:15:43,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:43,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T08:15:43,846 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436143845"}]},"ts":"1732436143845"} 2024-11-24T08:15:43,847 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T08:15:43,847 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T08:15:43,848 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T08:15:43,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, UNASSIGN}] 2024-11-24T08:15:43,850 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, UNASSIGN 2024-11-24T08:15:43,850 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=36db097a55561465c9ef689006e62ac4, regionState=CLOSING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:15:43,851 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, UNASSIGN because future has completed 2024-11-24T08:15:43,851 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T08:15:43,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:15:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T08:15:44,003 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:44,003 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T08:15:44,003 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 36db097a55561465c9ef689006e62ac4, disabling compactions & flushes 2024-11-24T08:15:44,004 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:44,057 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/86c06d2de8df4e0eb87cb9f063c2a294 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/86c06d2de8df4e0eb87cb9f063c2a294 2024-11-24T08:15:44,059 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/061a95ee25644d128348732e427c9893 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/061a95ee25644d128348732e427c9893 2024-11-24T08:15:44,061 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36db097a55561465c9ef689006e62ac4/C of 36db097a55561465c9ef689006e62ac4 into 86c06d2de8df4e0eb87cb9f063c2a294(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:44,061 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:44,061 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/C, priority=13, startTime=1732436143221; duration=0sec 2024-11-24T08:15:44,061 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:44,061 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:C 2024-11-24T08:15:44,062 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 36db097a55561465c9ef689006e62ac4/A of 36db097a55561465c9ef689006e62ac4 into 061a95ee25644d128348732e427c9893(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:44,062 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 36db097a55561465c9ef689006e62ac4: 2024-11-24T08:15:44,062 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., storeName=36db097a55561465c9ef689006e62ac4/A, priority=13, startTime=1732436143221; duration=0sec 2024-11-24T08:15:44,062 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:44,062 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:44,062 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 36db097a55561465c9ef689006e62ac4:A 2024-11-24T08:15:44,063 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:44,063 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. after waiting 0 ms 2024-11-24T08:15:44,063 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:44,063 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 36db097a55561465c9ef689006e62ac4 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-24T08:15:44,063 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=A 2024-11-24T08:15:44,063 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:44,063 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=B 2024-11-24T08:15:44,063 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:44,063 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 36db097a55561465c9ef689006e62ac4, store=C 2024-11-24T08:15:44,063 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:44,068 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245bc066c5b66141edb0f95a1b32993b24_36db097a55561465c9ef689006e62ac4 is 50, key is test_row_1/A:col10/1732436143840/Put/seqid=0 2024-11-24T08:15:44,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742251_1427 (size=9914) 2024-11-24T08:15:44,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T08:15:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T08:15:44,474 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:44,477 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411245bc066c5b66141edb0f95a1b32993b24_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245bc066c5b66141edb0f95a1b32993b24_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:44,478 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/4b1364822fcb4594bee73d5f16acb8df, store: [table=TestAcidGuarantees family=A region=36db097a55561465c9ef689006e62ac4] 2024-11-24T08:15:44,479 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/4b1364822fcb4594bee73d5f16acb8df is 175, key is test_row_1/A:col10/1732436143840/Put/seqid=0 2024-11-24T08:15:44,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742252_1428 (size=22561) 2024-11-24T08:15:44,885 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=431, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/4b1364822fcb4594bee73d5f16acb8df 2024-11-24T08:15:44,891 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/245dc7ae49ef4611b838f1e40dc956ba is 50, key is test_row_1/B:col10/1732436143840/Put/seqid=0 2024-11-24T08:15:44,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742253_1429 (size=9857) 2024-11-24T08:15:44,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T08:15:45,295 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/245dc7ae49ef4611b838f1e40dc956ba 2024-11-24T08:15:45,301 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/1695e6c19f6a4b4ea0cb7eb195f465db is 50, key is test_row_1/C:col10/1732436143840/Put/seqid=0 2024-11-24T08:15:45,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742254_1430 (size=9857) 2024-11-24T08:15:45,704 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/1695e6c19f6a4b4ea0cb7eb195f465db 2024-11-24T08:15:45,708 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/A/4b1364822fcb4594bee73d5f16acb8df as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/4b1364822fcb4594bee73d5f16acb8df 2024-11-24T08:15:45,711 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/4b1364822fcb4594bee73d5f16acb8df, entries=100, sequenceid=431, filesize=22.0 K 2024-11-24T08:15:45,712 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/B/245dc7ae49ef4611b838f1e40dc956ba as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/245dc7ae49ef4611b838f1e40dc956ba 2024-11-24T08:15:45,714 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/245dc7ae49ef4611b838f1e40dc956ba, entries=100, sequenceid=431, filesize=9.6 K 2024-11-24T08:15:45,715 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/.tmp/C/1695e6c19f6a4b4ea0cb7eb195f465db as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/1695e6c19f6a4b4ea0cb7eb195f465db 2024-11-24T08:15:45,718 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/1695e6c19f6a4b4ea0cb7eb195f465db, entries=100, sequenceid=431, filesize=9.6 K 2024-11-24T08:15:45,719 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 36db097a55561465c9ef689006e62ac4 in 1656ms, sequenceid=431, compaction requested=false 2024-11-24T08:15:45,719 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/4fa137f5e54c4925a2492e865701f0e9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/8636219dcb394c8bb19e7335af6c9ed8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/db2b14148bf9472f87ed64260068a8e6, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/dde716388f3a4eedbb2e52aecfa0f10f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/5cf33751e17c48caa93540324f4b603e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/267a2daefcc54ceba61dabba8ca77285, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/24d1f3d95428463ba7e27bbdb89c0a13] to archive 2024-11-24T08:15:45,720 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:15:45,721 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/4fa137f5e54c4925a2492e865701f0e9 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/4fa137f5e54c4925a2492e865701f0e9 2024-11-24T08:15:45,722 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/8636219dcb394c8bb19e7335af6c9ed8 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/8636219dcb394c8bb19e7335af6c9ed8 2024-11-24T08:15:45,723 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/db2b14148bf9472f87ed64260068a8e6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/db2b14148bf9472f87ed64260068a8e6 2024-11-24T08:15:45,724 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/dde716388f3a4eedbb2e52aecfa0f10f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/dde716388f3a4eedbb2e52aecfa0f10f 2024-11-24T08:15:45,725 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/5cf33751e17c48caa93540324f4b603e to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/5cf33751e17c48caa93540324f4b603e 2024-11-24T08:15:45,726 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/267a2daefcc54ceba61dabba8ca77285 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/267a2daefcc54ceba61dabba8ca77285 2024-11-24T08:15:45,727 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/24d1f3d95428463ba7e27bbdb89c0a13 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/24d1f3d95428463ba7e27bbdb89c0a13 2024-11-24T08:15:45,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/ab043d39cf7746f198c5a553965ca985, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/25e8b213a36244c3bb454ebeeb98bc98, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/60d3bcea687c4b2e82ba48f71530fe57, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/f1af1126a0414b23b622aba73a86f42a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/80c63b6197be45db8a66fd2734981f92, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/ddd3b3fbe10144ee859681c440086b18, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/57507be8b5c343038a829770ef38f34b] to archive 2024-11-24T08:15:45,728 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:15:45,729 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/ab043d39cf7746f198c5a553965ca985 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/ab043d39cf7746f198c5a553965ca985 2024-11-24T08:15:45,730 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/25e8b213a36244c3bb454ebeeb98bc98 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/25e8b213a36244c3bb454ebeeb98bc98 2024-11-24T08:15:45,731 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/60d3bcea687c4b2e82ba48f71530fe57 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/60d3bcea687c4b2e82ba48f71530fe57 2024-11-24T08:15:45,732 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/f1af1126a0414b23b622aba73a86f42a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/f1af1126a0414b23b622aba73a86f42a 2024-11-24T08:15:45,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/80c63b6197be45db8a66fd2734981f92 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/80c63b6197be45db8a66fd2734981f92 2024-11-24T08:15:45,733 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/ddd3b3fbe10144ee859681c440086b18 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/ddd3b3fbe10144ee859681c440086b18 2024-11-24T08:15:45,734 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/57507be8b5c343038a829770ef38f34b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/57507be8b5c343038a829770ef38f34b 2024-11-24T08:15:45,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/07d248faf9984eaba4457245a92f1aef, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/478b95545a50451193d543b9e249b1fa, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/77e294b099c047dcbf37edee62f07cb1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/6eedbee977404f479f5dc3e159015d60, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/99b5077a927d4b91bd0e556b158f3cec, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/fde1103e726748f9ba857ea90111b12e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/4b30384fc93e44e184147286f9c08406] to archive 2024-11-24T08:15:45,735 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:15:45,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/07d248faf9984eaba4457245a92f1aef to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/07d248faf9984eaba4457245a92f1aef 2024-11-24T08:15:45,737 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/478b95545a50451193d543b9e249b1fa to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/478b95545a50451193d543b9e249b1fa 2024-11-24T08:15:45,738 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/77e294b099c047dcbf37edee62f07cb1 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/77e294b099c047dcbf37edee62f07cb1 2024-11-24T08:15:45,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/6eedbee977404f479f5dc3e159015d60 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/6eedbee977404f479f5dc3e159015d60 2024-11-24T08:15:45,739 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/99b5077a927d4b91bd0e556b158f3cec to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/99b5077a927d4b91bd0e556b158f3cec 2024-11-24T08:15:45,740 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/fde1103e726748f9ba857ea90111b12e to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/fde1103e726748f9ba857ea90111b12e 2024-11-24T08:15:45,741 DEBUG [StoreCloser-TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/4b30384fc93e44e184147286f9c08406 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/4b30384fc93e44e184147286f9c08406 2024-11-24T08:15:45,744 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/recovered.edits/434.seqid, newMaxSeqId=434, maxSeqId=4 2024-11-24T08:15:45,745 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. 2024-11-24T08:15:45,745 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 36db097a55561465c9ef689006e62ac4: Waiting for close lock at 1732436144003Running coprocessor pre-close hooks at 1732436144003Disabling compacts and flushes for region at 1732436144003Disabling writes for close at 1732436144063 (+60 ms)Obtaining lock to block concurrent updates at 1732436144063Preparing flush snapshotting stores in 36db097a55561465c9ef689006e62ac4 at 1732436144063Finished memstore snapshotting TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1732436144063Flushing stores of TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4. at 1732436144064 (+1 ms)Flushing 36db097a55561465c9ef689006e62ac4/A: creating writer at 1732436144064Flushing mob file 36db097a55561465c9ef689006e62ac4/A: appending metadata at 1732436144068 (+4 ms)Flushing mob file 36db097a55561465c9ef689006e62ac4/A: closing flushed file at 1732436144068Flushing 36db097a55561465c9ef689006e62ac4/A: appending metadata at 1732436144478 (+410 ms)Flushing 36db097a55561465c9ef689006e62ac4/A: closing flushed file at 1732436144478Flushing 36db097a55561465c9ef689006e62ac4/B: creating writer at 1732436144889 (+411 ms)Flushing 36db097a55561465c9ef689006e62ac4/B: appending metadata at 1732436144891 (+2 ms)Flushing 36db097a55561465c9ef689006e62ac4/B: closing flushed file at 1732436144891Flushing 36db097a55561465c9ef689006e62ac4/C: creating writer at 1732436145298 (+407 ms)Flushing 36db097a55561465c9ef689006e62ac4/C: appending metadata at 1732436145300 (+2 ms)Flushing 36db097a55561465c9ef689006e62ac4/C: closing flushed file at 1732436145300Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@658e07af: reopening flushed file at 1732436145707 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@426e442f: reopening flushed file at 1732436145711 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ce947a8: reopening flushed file at 1732436145715 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 36db097a55561465c9ef689006e62ac4 in 1656ms, sequenceid=431, compaction requested=false at 1732436145719 (+4 ms)Writing region close event to WAL at 1732436145742 (+23 ms)Running coprocessor post-close hooks at 1732436145745 (+3 ms)Closed at 1732436145745 2024-11-24T08:15:45,746 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:45,747 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=36db097a55561465c9ef689006e62ac4, regionState=CLOSED 2024-11-24T08:15:45,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:15:45,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-24T08:15:45,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 36db097a55561465c9ef689006e62ac4, server=30c28c82771d,45087,1732436016722 in 1.8970 sec 2024-11-24T08:15:45,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-24T08:15:45,751 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=36db097a55561465c9ef689006e62ac4, UNASSIGN in 1.9010 sec 2024-11-24T08:15:45,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-24T08:15:45,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9040 sec 2024-11-24T08:15:45,754 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436145754"}]},"ts":"1732436145754"} 2024-11-24T08:15:45,755 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T08:15:45,755 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T08:15:45,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9120 sec 2024-11-24T08:15:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-24T08:15:45,990 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:45,990 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T08:15:45,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:45,992 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:45,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T08:15:45,992 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:45,995 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:45,997 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/recovered.edits] 2024-11-24T08:15:45,999 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/061a95ee25644d128348732e427c9893 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/061a95ee25644d128348732e427c9893 2024-11-24T08:15:45,999 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/4b1364822fcb4594bee73d5f16acb8df to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/A/4b1364822fcb4594bee73d5f16acb8df 2024-11-24T08:15:46,001 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/245dc7ae49ef4611b838f1e40dc956ba to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/245dc7ae49ef4611b838f1e40dc956ba 2024-11-24T08:15:46,002 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/c345dadb4c1b4ec2ad1a49efec4c699d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/B/c345dadb4c1b4ec2ad1a49efec4c699d 2024-11-24T08:15:46,004 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/1695e6c19f6a4b4ea0cb7eb195f465db to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/1695e6c19f6a4b4ea0cb7eb195f465db 2024-11-24T08:15:46,005 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/86c06d2de8df4e0eb87cb9f063c2a294 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/C/86c06d2de8df4e0eb87cb9f063c2a294 2024-11-24T08:15:46,006 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/recovered.edits/434.seqid to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4/recovered.edits/434.seqid 2024-11-24T08:15:46,007 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,007 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T08:15:46,007 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T08:15:46,008 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T08:15:46,010 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112404311178ca4549a3a1163c4491b906fd_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112404311178ca4549a3a1163c4491b906fd_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,011 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124052a9e71d2654c7488d44d06fef581f2_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124052a9e71d2654c7488d44d06fef581f2_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,012 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240824f65ba0fb4661816bd3087db4ad93_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240824f65ba0fb4661816bd3087db4ad93_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,013 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240c218df0a27541f39559f4f302f9fb9c_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411240c218df0a27541f39559f4f302f9fb9c_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,013 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243f750689bcef43799710ba937f3bd7a6_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411243f750689bcef43799710ba937f3bd7a6_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,014 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244b39aa2a33dd443aaaaadd896d31f89f_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244b39aa2a33dd443aaaaadd896d31f89f_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,015 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244c2711425efb44a6b331d8b7bb44259f_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244c2711425efb44a6b331d8b7bb44259f_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,016 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245bc066c5b66141edb0f95a1b32993b24_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411245bc066c5b66141edb0f95a1b32993b24_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,017 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124620aa0dea24d4a69858a1b21303b7a44_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124620aa0dea24d4a69858a1b21303b7a44_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,017 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246f0747936e3e48958de5fbbbc1990957_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411246f0747936e3e48958de5fbbbc1990957_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,018 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248e6490def62145fda08f8b618f8df1b9_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411248e6490def62145fda08f8b618f8df1b9_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,019 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249af62818a23d4860a57b84a1c43a79d2_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249af62818a23d4860a57b84a1c43a79d2_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,020 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249f6d08fe17eb411abae56e51f11707a6_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411249f6d08fe17eb411abae56e51f11707a6_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,021 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124aebf769a4c3c4d458f6f12acaed67854_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124aebf769a4c3c4d458f6f12acaed67854_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,022 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b9069e4eb9ca403aaacca1d2846833a1_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124b9069e4eb9ca403aaacca1d2846833a1_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,023 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bb28332e1cbe4ab7baf56fc101c6cc2e_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bb28332e1cbe4ab7baf56fc101c6cc2e_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,024 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d3ace590b0064d5eae592d1824466ebb_36db097a55561465c9ef689006e62ac4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d3ace590b0064d5eae592d1824466ebb_36db097a55561465c9ef689006e62ac4 2024-11-24T08:15:46,024 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T08:15:46,026 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:46,028 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T08:15:46,030 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T08:15:46,031 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:46,031 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T08:15:46,031 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732436146031"}]},"ts":"9223372036854775807"} 2024-11-24T08:15:46,033 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T08:15:46,033 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 36db097a55561465c9ef689006e62ac4, NAME => 'TestAcidGuarantees,,1732436117389.36db097a55561465c9ef689006e62ac4.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T08:15:46,033 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T08:15:46,033 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732436146033"}]},"ts":"9223372036854775807"} 2024-11-24T08:15:46,035 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T08:15:46,036 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:46,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-24T08:15:46,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-24T08:15:46,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T08:15:46,101 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:46,110 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=254 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-921854773_22 at /127.0.0.1:34612 [Waiting for operation #816] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_554043647_22 at /127.0.0.1:49952 [Waiting for operation #906] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-921854773_22 at /127.0.0.1:37472 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_554043647_22 at /127.0.0.1:34624 [Waiting for operation #830] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/30c28c82771d:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_554043647_22 at /127.0.0.1:34524 [Waiting for operation #857] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=294 (was 314), ProcessCount=11 (was 11), AvailableMemoryMB=6318 (was 6798) 2024-11-24T08:15:46,118 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=254, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=294, ProcessCount=11, AvailableMemoryMB=6318 2024-11-24T08:15:46,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T08:15:46,120 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:15:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T08:15:46,121 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T08:15:46,122 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:46,122 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-24T08:15:46,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T08:15:46,122 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T08:15:46,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742255_1431 (size=1061) 2024-11-24T08:15:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T08:15:46,300 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T08:15:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T08:15:46,529 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6 2024-11-24T08:15:46,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742256_1432 (size=53) 2024-11-24T08:15:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T08:15:46,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:15:46,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a55c3d95315c8337b2d7386eb1a0fc85, disabling compactions & flushes 2024-11-24T08:15:46,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:46,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:46,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. after waiting 0 ms 2024-11-24T08:15:46,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:46,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:46,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a55c3d95315c8337b2d7386eb1a0fc85: Waiting for close lock at 1732436146934Disabling compacts and flushes for region at 1732436146934Disabling writes for close at 1732436146934Writing region close event to WAL at 1732436146934Closed at 1732436146934 2024-11-24T08:15:46,935 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T08:15:46,935 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732436146935"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732436146935"}]},"ts":"1732436146935"} 2024-11-24T08:15:46,937 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T08:15:46,937 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T08:15:46,937 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436146937"}]},"ts":"1732436146937"} 2024-11-24T08:15:46,938 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T08:15:46,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a55c3d95315c8337b2d7386eb1a0fc85, ASSIGN}] 2024-11-24T08:15:46,939 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a55c3d95315c8337b2d7386eb1a0fc85, ASSIGN 2024-11-24T08:15:46,940 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a55c3d95315c8337b2d7386eb1a0fc85, ASSIGN; state=OFFLINE, location=30c28c82771d,45087,1732436016722; forceNewPlan=false, retain=false 2024-11-24T08:15:47,091 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=a55c3d95315c8337b2d7386eb1a0fc85, regionState=OPENING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:15:47,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a55c3d95315c8337b2d7386eb1a0fc85, ASSIGN because future has completed 2024-11-24T08:15:47,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:15:47,246 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:47,247 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:15:47,247 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:47,247 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:15:47,247 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:47,247 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:47,249 INFO [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:47,250 INFO [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:15:47,250 INFO [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a55c3d95315c8337b2d7386eb1a0fc85 columnFamilyName A 2024-11-24T08:15:47,250 DEBUG [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:47,250 INFO [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] regionserver.HStore(327): Store=a55c3d95315c8337b2d7386eb1a0fc85/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:15:47,251 INFO [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:47,251 INFO [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:15:47,251 INFO [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a55c3d95315c8337b2d7386eb1a0fc85 columnFamilyName B 2024-11-24T08:15:47,252 DEBUG [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:47,252 INFO [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] regionserver.HStore(327): Store=a55c3d95315c8337b2d7386eb1a0fc85/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:15:47,252 INFO [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:47,253 INFO [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:15:47,253 INFO [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a55c3d95315c8337b2d7386eb1a0fc85 columnFamilyName C 2024-11-24T08:15:47,253 DEBUG [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:15:47,253 INFO [StoreOpener-a55c3d95315c8337b2d7386eb1a0fc85-1 {}] regionserver.HStore(327): Store=a55c3d95315c8337b2d7386eb1a0fc85/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:15:47,253 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:47,253 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:47,254 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:47,254 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:47,255 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:47,255 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:47,255 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T08:15:47,256 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:47,257 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:15:47,258 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened a55c3d95315c8337b2d7386eb1a0fc85; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66098998, jitterRate=-0.015048176050186157}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T08:15:47,258 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:47,258 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for a55c3d95315c8337b2d7386eb1a0fc85: Running coprocessor pre-open hook at 1732436147248Writing region info on filesystem at 1732436147248Initializing all the Stores at 1732436147248Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436147248Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436147248Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436147248Cleaning up temporary data from old regions at 1732436147255 (+7 ms)Running coprocessor post-open hooks at 1732436147258 (+3 ms)Region opened successfully at 1732436147258 2024-11-24T08:15:47,259 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., pid=120, masterSystemTime=1732436147244 2024-11-24T08:15:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T08:15:47,260 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:47,260 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:47,261 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=a55c3d95315c8337b2d7386eb1a0fc85, regionState=OPEN, openSeqNum=2, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:15:47,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:15:47,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-24T08:15:47,264 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 in 170 msec 2024-11-24T08:15:47,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-24T08:15:47,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a55c3d95315c8337b2d7386eb1a0fc85, ASSIGN in 326 msec 2024-11-24T08:15:47,266 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T08:15:47,266 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436147266"}]},"ts":"1732436147266"} 2024-11-24T08:15:47,267 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T08:15:47,267 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T08:15:47,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-24T08:15:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-24T08:15:48,280 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:48,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd122f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,281 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:48,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:48,282 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36356b1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:48,284 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:48,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135e79b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,285 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:48,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:48,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:48,286 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T08:15:48,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 30c28c82771d,38469,1732436015766 2024-11-24T08:15:48,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@352ab98 2024-11-24T08:15:48,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T08:15:48,288 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T08:15:48,289 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,289 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:48,289 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:48,289 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594f7109, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:48,291 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:48,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d180957, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,291 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:48,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:48,292 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c9bc25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:48,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:48,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:48,293 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:48,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@446f3105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:48,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:48,295 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b1da4a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:48,296 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:48,296 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f427b9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,296 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:48,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:48,297 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21851c27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:48,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:48,298 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:48,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e4c52dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:48,299 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:48,300 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b024cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:48,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:48,301 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:48,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3efe8ac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,301 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,301 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:48,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:48,302 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:48,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:48,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:48,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a0f7217, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:48,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:48,303 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:48,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66dc17d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:48,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:48,304 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337087e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:48,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:48,305 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:48,306 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dded0dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,306 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:48,306 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:48,307 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:48,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:48,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:48,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5accbabb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:48,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:48,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:48,308 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a88053f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:48,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:48,309 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:48,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@499c577f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:48,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:48,310 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:48,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aee36a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bb1bf9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:15:48,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:15:48,312 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:15:48,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:15:48,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:15:48,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@438bbe5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:15:48,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:15:48,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:15:48,313 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:15:48,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@606c0d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:15:48,313 DEBUG [Thread-1961 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:48,313 DEBUG [Thread-1950 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:48,314 DEBUG [Thread-1956 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:48,314 DEBUG [Thread-1963 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:48,314 DEBUG [Thread-1948 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:48,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:48,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:48,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:48,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:48,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:48,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:48,315 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:48,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:48,316 DEBUG [Thread-1965 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:48,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:48,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:48,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:48,316 DEBUG [Thread-1967 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:48,317 DEBUG [Thread-1952 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-24T08:15:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:48,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:15:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:15:48,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:15:48,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T08:15:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:48,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:48,319 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:48,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:48,320 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:48,320 DEBUG [Thread-1954 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:15:48,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:15:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:15:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:15:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:48,322 DEBUG [Thread-1959 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:15:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:48,322 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:48,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:48,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:15:48,324 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:15:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:15:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:15:48,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:15:48,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2] 2024-11-24T08:15:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:48,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:15:48,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:15:48,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:48,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:15:48,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:48,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:15:48,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:48,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/93a8aabc62de4782a24a915167c4eb3b is 50, key is test_row_0/A:col10/1732436148329/Put/seqid=0 2024-11-24T08:15:48,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742257_1433 (size=16681) 2024-11-24T08:15:48,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436208355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436208366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436208367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436208369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436208369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T08:15:48,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T08:15:48,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:48,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:48,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:48,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:48,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:48,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436208480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436208480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436208480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436208490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436208490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T08:15:48,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:48,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:48,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:48,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:48,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:48,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T08:15:48,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436208700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436208700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436208700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436208700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436208700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:48,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/93a8aabc62de4782a24a915167c4eb3b 2024-11-24T08:15:48,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/fe21f0ba34574846bd821e2c88263c7b is 50, key is test_row_0/B:col10/1732436148329/Put/seqid=0 2024-11-24T08:15:48,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742258_1434 (size=12001) 2024-11-24T08:15:48,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T08:15:48,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:48,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:48,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:48,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:48,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:48,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:48,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T08:15:48,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:48,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:48,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:48,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:48,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:48,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T08:15:49,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436209009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:49,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436209010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:49,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436209010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:49,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436209010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:49,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436209020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:49,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T08:15:49,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:49,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:49,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:49,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:49,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:49,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:49,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/fe21f0ba34574846bd821e2c88263c7b 2024-11-24T08:15:49,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/82289d025ede4feca0419acd36c9eecd is 50, key is test_row_0/C:col10/1732436148329/Put/seqid=0 2024-11-24T08:15:49,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742259_1435 (size=12001) 2024-11-24T08:15:49,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T08:15:49,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:49,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:49,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:49,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:49,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:49,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T08:15:49,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:49,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:49,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:49,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:49,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T08:15:49,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436209520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:49,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436209520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436209520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:49,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436209520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:49,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436209529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:49,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:49,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T08:15:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:49,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:49,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:49,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:49,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:49,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/82289d025ede4feca0419acd36c9eecd 2024-11-24T08:15:49,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/93a8aabc62de4782a24a915167c4eb3b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/93a8aabc62de4782a24a915167c4eb3b 2024-11-24T08:15:49,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/93a8aabc62de4782a24a915167c4eb3b, entries=250, sequenceid=13, filesize=16.3 K 2024-11-24T08:15:49,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/fe21f0ba34574846bd821e2c88263c7b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/fe21f0ba34574846bd821e2c88263c7b 2024-11-24T08:15:49,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/fe21f0ba34574846bd821e2c88263c7b, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T08:15:49,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/82289d025ede4feca0419acd36c9eecd as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/82289d025ede4feca0419acd36c9eecd 2024-11-24T08:15:49,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/82289d025ede4feca0419acd36c9eecd, entries=150, sequenceid=13, filesize=11.7 K 2024-11-24T08:15:49,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a55c3d95315c8337b2d7386eb1a0fc85 in 1300ms, sequenceid=13, compaction requested=false 2024-11-24T08:15:49,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:49,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-24T08:15:49,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:49,701 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T08:15:49,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:15:49,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:49,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:15:49,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:49,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:15:49,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:49,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/ebcf0c8d11ff4a149ce8521d220c9aa0 is 50, key is test_row_0/A:col10/1732436148364/Put/seqid=0 2024-11-24T08:15:49,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742260_1436 (size=12001) 2024-11-24T08:15:50,114 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/ebcf0c8d11ff4a149ce8521d220c9aa0 2024-11-24T08:15:50,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/9d9c21c843ad4440bf275997d65d2556 is 50, key is test_row_0/B:col10/1732436148364/Put/seqid=0 2024-11-24T08:15:50,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742261_1437 (size=12001) 2024-11-24T08:15:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T08:15:50,528 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/9d9c21c843ad4440bf275997d65d2556 2024-11-24T08:15:50,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/2fa5774251184cf7872960a338e11f9f is 50, key is test_row_0/C:col10/1732436148364/Put/seqid=0 2024-11-24T08:15:50,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742262_1438 (size=12001) 2024-11-24T08:15:50,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:50,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:50,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436210558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436210559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,562 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436210560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436210560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436210561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436210669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436210669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436210680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436210680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436210680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436210889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436210890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436210900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436210900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436210900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:50,939 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/2fa5774251184cf7872960a338e11f9f 2024-11-24T08:15:50,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/ebcf0c8d11ff4a149ce8521d220c9aa0 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/ebcf0c8d11ff4a149ce8521d220c9aa0 2024-11-24T08:15:50,946 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/ebcf0c8d11ff4a149ce8521d220c9aa0, entries=150, sequenceid=33, filesize=11.7 K 2024-11-24T08:15:50,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/9d9c21c843ad4440bf275997d65d2556 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/9d9c21c843ad4440bf275997d65d2556 2024-11-24T08:15:50,957 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/9d9c21c843ad4440bf275997d65d2556, entries=150, sequenceid=33, filesize=11.7 K 2024-11-24T08:15:50,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/2fa5774251184cf7872960a338e11f9f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/2fa5774251184cf7872960a338e11f9f 2024-11-24T08:15:50,963 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/2fa5774251184cf7872960a338e11f9f, entries=150, sequenceid=33, filesize=11.7 K 2024-11-24T08:15:50,964 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a55c3d95315c8337b2d7386eb1a0fc85 in 1264ms, sequenceid=33, compaction requested=false 2024-11-24T08:15:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-24T08:15:50,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-24T08:15:50,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-24T08:15:50,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6440 sec 2024-11-24T08:15:50,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.6550 sec 2024-11-24T08:15:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:51,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T08:15:51,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:15:51,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:51,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:15:51,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:51,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:15:51,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:51,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/2a75e9a702044926a2245d29442fd5ec is 50, key is test_row_0/A:col10/1732436150558/Put/seqid=0 2024-11-24T08:15:51,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742263_1439 (size=14341) 2024-11-24T08:15:51,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436211240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,255 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436211253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,261 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T08:15:51,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436211255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436211256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436211256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436211359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436211359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436211380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436211380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436211381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436211580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436211580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436211590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436211590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436211599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/2a75e9a702044926a2245d29442fd5ec 2024-11-24T08:15:51,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/4523bd19e34c48d8928eeefcca38f249 is 50, key is test_row_0/B:col10/1732436150558/Put/seqid=0 2024-11-24T08:15:51,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742264_1440 (size=12001) 2024-11-24T08:15:51,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436211889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,899 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436211899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436211910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436211910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436211910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:51,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:52,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/4523bd19e34c48d8928eeefcca38f249 2024-11-24T08:15:52,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/c6ae60adcc97407085ae08938c3f5454 is 50, key is test_row_0/C:col10/1732436150558/Put/seqid=0 2024-11-24T08:15:52,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742265_1441 (size=12001) 2024-11-24T08:15:52,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436212409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:52,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:52,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:52,414 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:52,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436212420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:52,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:52,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:52,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:52,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436212430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:52,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436212430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:52,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:52,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436212430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:52,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:52,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/c6ae60adcc97407085ae08938c3f5454 2024-11-24T08:15:52,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/2a75e9a702044926a2245d29442fd5ec as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/2a75e9a702044926a2245d29442fd5ec 2024-11-24T08:15:52,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/2a75e9a702044926a2245d29442fd5ec, entries=200, sequenceid=46, filesize=14.0 K 2024-11-24T08:15:52,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/4523bd19e34c48d8928eeefcca38f249 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4523bd19e34c48d8928eeefcca38f249 2024-11-24T08:15:52,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4523bd19e34c48d8928eeefcca38f249, entries=150, sequenceid=46, filesize=11.7 K 2024-11-24T08:15:52,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/c6ae60adcc97407085ae08938c3f5454 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/c6ae60adcc97407085ae08938c3f5454 2024-11-24T08:15:52,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/c6ae60adcc97407085ae08938c3f5454, entries=150, sequenceid=46, filesize=11.7 K 2024-11-24T08:15:52,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for a55c3d95315c8337b2d7386eb1a0fc85 in 1253ms, sequenceid=46, compaction requested=true 2024-11-24T08:15:52,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:52,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:52,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:52,459 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:52,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:52,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:52,460 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:52,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:52,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:52,460 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:52,461 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:52,461 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/A is initiating minor compaction (all files) 2024-11-24T08:15:52,461 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/B is initiating minor compaction (all files) 2024-11-24T08:15:52,461 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/A in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:52,461 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/B in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:52,461 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/fe21f0ba34574846bd821e2c88263c7b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/9d9c21c843ad4440bf275997d65d2556, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4523bd19e34c48d8928eeefcca38f249] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=35.2 K 2024-11-24T08:15:52,461 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/93a8aabc62de4782a24a915167c4eb3b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/ebcf0c8d11ff4a149ce8521d220c9aa0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/2a75e9a702044926a2245d29442fd5ec] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=42.0 K 2024-11-24T08:15:52,461 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting fe21f0ba34574846bd821e2c88263c7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732436148319 2024-11-24T08:15:52,461 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93a8aabc62de4782a24a915167c4eb3b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732436148319 2024-11-24T08:15:52,461 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d9c21c843ad4440bf275997d65d2556, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732436148350 2024-11-24T08:15:52,461 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebcf0c8d11ff4a149ce8521d220c9aa0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732436148350 2024-11-24T08:15:52,462 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4523bd19e34c48d8928eeefcca38f249, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732436150558 2024-11-24T08:15:52,462 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a75e9a702044926a2245d29442fd5ec, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732436150558 2024-11-24T08:15:52,469 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#B#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:52,470 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/0b99df2245d044b294f0394388521d11 is 50, key is test_row_0/B:col10/1732436150558/Put/seqid=0 2024-11-24T08:15:52,476 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#A#compaction#367 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:52,477 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/21ced72d0d734ac09ff61515da492a8e is 50, key is test_row_0/A:col10/1732436150558/Put/seqid=0 2024-11-24T08:15:52,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742266_1442 (size=12104) 2024-11-24T08:15:52,483 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/0b99df2245d044b294f0394388521d11 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/0b99df2245d044b294f0394388521d11 2024-11-24T08:15:52,487 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/B of a55c3d95315c8337b2d7386eb1a0fc85 into 0b99df2245d044b294f0394388521d11(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:52,487 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:52,487 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/B, priority=13, startTime=1732436152459; duration=0sec 2024-11-24T08:15:52,487 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:52,487 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:B 2024-11-24T08:15:52,487 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:52,488 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:52,488 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/C is initiating minor compaction (all files) 2024-11-24T08:15:52,489 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/C in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:52,489 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/82289d025ede4feca0419acd36c9eecd, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/2fa5774251184cf7872960a338e11f9f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/c6ae60adcc97407085ae08938c3f5454] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=35.2 K 2024-11-24T08:15:52,489 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 82289d025ede4feca0419acd36c9eecd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732436148319 2024-11-24T08:15:52,489 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fa5774251184cf7872960a338e11f9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732436148350 2024-11-24T08:15:52,489 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting c6ae60adcc97407085ae08938c3f5454, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732436150558 2024-11-24T08:15:52,495 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#C#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:52,495 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/4e61f6442ca44f2a8bcaade410eebc99 is 50, key is test_row_0/C:col10/1732436150558/Put/seqid=0 2024-11-24T08:15:52,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742267_1443 (size=12104) 2024-11-24T08:15:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-24T08:15:52,500 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:52,501 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-24T08:15:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T08:15:52,503 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/21ced72d0d734ac09ff61515da492a8e as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/21ced72d0d734ac09ff61515da492a8e 2024-11-24T08:15:52,507 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:52,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742268_1444 (size=12104) 2024-11-24T08:15:52,509 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:52,509 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:52,509 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/A of a55c3d95315c8337b2d7386eb1a0fc85 into 21ced72d0d734ac09ff61515da492a8e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:52,509 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:52,509 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/A, priority=13, startTime=1732436152459; duration=0sec 2024-11-24T08:15:52,509 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:52,509 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:A 2024-11-24T08:15:52,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T08:15:52,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-24T08:15:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:52,664 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T08:15:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:15:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:15:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:15:52,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:52,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/cf3a6db3da2e4ea29bc2bc66b1d74b32 is 50, key is test_row_0/A:col10/1732436151252/Put/seqid=0 2024-11-24T08:15:52,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742269_1445 (size=12001) 2024-11-24T08:15:52,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T08:15:52,913 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/4e61f6442ca44f2a8bcaade410eebc99 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/4e61f6442ca44f2a8bcaade410eebc99 2024-11-24T08:15:52,917 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/C of a55c3d95315c8337b2d7386eb1a0fc85 into 4e61f6442ca44f2a8bcaade410eebc99(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:52,917 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:52,917 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/C, priority=13, startTime=1732436152460; duration=0sec 2024-11-24T08:15:52,917 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:52,917 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:C 2024-11-24T08:15:53,074 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/cf3a6db3da2e4ea29bc2bc66b1d74b32 2024-11-24T08:15:53,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/367f916ab9e94ae2a48a701ce1f5531f is 50, key is test_row_0/B:col10/1732436151252/Put/seqid=0 2024-11-24T08:15:53,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742270_1446 (size=12001) 2024-11-24T08:15:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T08:15:53,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:53,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:53,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:53,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436213450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436213450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:53,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436213452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:53,461 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in a55c3d95315c8337b2d7386eb1a0fc85 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:53,461 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_0' in a55c3d95315c8337b2d7386eb1a0fc85 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:53,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:53,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436213461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:53,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436213461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:53,485 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/367f916ab9e94ae2a48a701ce1f5531f 2024-11-24T08:15:53,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/8082abc7ace1486badb22a7a99708b84 is 50, key is test_row_0/C:col10/1732436151252/Put/seqid=0 2024-11-24T08:15:53,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742271_1447 (size=12001) 2024-11-24T08:15:53,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436213570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:53,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436213570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:53,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436213570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T08:15:53,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436213790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:53,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436213790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:53,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,800 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:53,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:53,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436213790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:53,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:53,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:53,896 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/8082abc7ace1486badb22a7a99708b84 2024-11-24T08:15:53,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/cf3a6db3da2e4ea29bc2bc66b1d74b32 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/cf3a6db3da2e4ea29bc2bc66b1d74b32 2024-11-24T08:15:53,903 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/cf3a6db3da2e4ea29bc2bc66b1d74b32, entries=150, sequenceid=68, filesize=11.7 K 2024-11-24T08:15:53,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/367f916ab9e94ae2a48a701ce1f5531f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/367f916ab9e94ae2a48a701ce1f5531f 2024-11-24T08:15:53,906 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/367f916ab9e94ae2a48a701ce1f5531f, entries=150, sequenceid=68, filesize=11.7 K 2024-11-24T08:15:53,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/8082abc7ace1486badb22a7a99708b84 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8082abc7ace1486badb22a7a99708b84 2024-11-24T08:15:53,909 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8082abc7ace1486badb22a7a99708b84, entries=150, sequenceid=68, filesize=11.7 K 2024-11-24T08:15:53,910 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a55c3d95315c8337b2d7386eb1a0fc85 in 1246ms, sequenceid=68, compaction requested=false 2024-11-24T08:15:53,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:53,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:53,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-24T08:15:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-24T08:15:53,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-24T08:15:53,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4030 sec 2024-11-24T08:15:53,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.4130 sec 2024-11-24T08:15:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:54,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T08:15:54,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:15:54,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:54,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:15:54,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:54,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:15:54,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:54,122 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/f88946cbb7b74a1ca884f48a555ec22a is 50, key is test_row_0/A:col10/1732436154115/Put/seqid=0 2024-11-24T08:15:54,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742272_1448 (size=12001) 2024-11-24T08:15:54,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:54,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436214188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:54,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:54,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436214189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:54,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:54,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436214189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:54,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:54,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436214299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:54,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:54,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436214310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:54,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:54,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436214310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:54,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/f88946cbb7b74a1ca884f48a555ec22a 2024-11-24T08:15:54,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436214520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:54,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436214520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:54,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436214520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:54,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/3e9f5abfb0be42299ecf2d64345997dc is 50, key is test_row_0/B:col10/1732436154115/Put/seqid=0 2024-11-24T08:15:54,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742273_1449 (size=12001) 2024-11-24T08:15:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-24T08:15:54,650 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:54,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-24T08:15:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T08:15:54,652 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:54,653 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:54,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T08:15:54,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T08:15:54,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:54,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:54,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:54,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:54,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:54,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436214840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:54,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436214840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:54,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436214840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:54,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/3e9f5abfb0be42299ecf2d64345997dc 2024-11-24T08:15:54,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/cc306666cff444c9bece8cf3b425b207 is 50, key is test_row_0/C:col10/1732436154115/Put/seqid=0 2024-11-24T08:15:54,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742274_1450 (size=12001) 2024-11-24T08:15:54,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/cc306666cff444c9bece8cf3b425b207 2024-11-24T08:15:54,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T08:15:54,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:54,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:54,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:54,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:54,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:54,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/f88946cbb7b74a1ca884f48a555ec22a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/f88946cbb7b74a1ca884f48a555ec22a 2024-11-24T08:15:54,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/f88946cbb7b74a1ca884f48a555ec22a, entries=150, sequenceid=83, filesize=11.7 K 2024-11-24T08:15:54,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/3e9f5abfb0be42299ecf2d64345997dc as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3e9f5abfb0be42299ecf2d64345997dc 2024-11-24T08:15:54,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T08:15:54,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3e9f5abfb0be42299ecf2d64345997dc, entries=150, sequenceid=83, filesize=11.7 K 2024-11-24T08:15:54,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/cc306666cff444c9bece8cf3b425b207 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/cc306666cff444c9bece8cf3b425b207 2024-11-24T08:15:54,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/cc306666cff444c9bece8cf3b425b207, entries=150, sequenceid=83, filesize=11.7 K 2024-11-24T08:15:54,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for a55c3d95315c8337b2d7386eb1a0fc85 in 859ms, sequenceid=83, compaction requested=true 2024-11-24T08:15:54,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:54,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:54,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:54,975 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:54,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:54,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:54,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:54,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:54,975 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:54,976 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:54,976 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/A is initiating minor compaction (all files) 2024-11-24T08:15:54,976 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/A in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:54,976 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/21ced72d0d734ac09ff61515da492a8e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/cf3a6db3da2e4ea29bc2bc66b1d74b32, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/f88946cbb7b74a1ca884f48a555ec22a] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=35.3 K 2024-11-24T08:15:54,976 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:54,976 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/B is initiating minor compaction (all files) 2024-11-24T08:15:54,976 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21ced72d0d734ac09ff61515da492a8e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732436150558 2024-11-24T08:15:54,976 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/B in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:54,976 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/0b99df2245d044b294f0394388521d11, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/367f916ab9e94ae2a48a701ce1f5531f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3e9f5abfb0be42299ecf2d64345997dc] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=35.3 K 2024-11-24T08:15:54,977 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b99df2245d044b294f0394388521d11, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732436150558 2024-11-24T08:15:54,977 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf3a6db3da2e4ea29bc2bc66b1d74b32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732436151252 2024-11-24T08:15:54,977 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 367f916ab9e94ae2a48a701ce1f5531f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732436151252 2024-11-24T08:15:54,977 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f88946cbb7b74a1ca884f48a555ec22a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732436153459 2024-11-24T08:15:54,978 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e9f5abfb0be42299ecf2d64345997dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732436153459 2024-11-24T08:15:54,983 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#B#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:54,984 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:54,984 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/3edb9f22e4524ded812de8dc6b4a35da is 50, key is test_row_0/B:col10/1732436154115/Put/seqid=0 2024-11-24T08:15:54,984 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/6401a261a34c40da997d9e0ed86f58a7 is 50, key is test_row_0/A:col10/1732436154115/Put/seqid=0 2024-11-24T08:15:54,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742276_1452 (size=12207) 2024-11-24T08:15:54,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742275_1451 (size=12207) 2024-11-24T08:15:55,000 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/6401a261a34c40da997d9e0ed86f58a7 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/6401a261a34c40da997d9e0ed86f58a7 2024-11-24T08:15:55,000 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/3edb9f22e4524ded812de8dc6b4a35da as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3edb9f22e4524ded812de8dc6b4a35da 2024-11-24T08:15:55,005 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/A of a55c3d95315c8337b2d7386eb1a0fc85 into 6401a261a34c40da997d9e0ed86f58a7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:55,005 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:55,005 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/A, priority=13, startTime=1732436154975; duration=0sec 2024-11-24T08:15:55,005 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/B of a55c3d95315c8337b2d7386eb1a0fc85 into 3edb9f22e4524ded812de8dc6b4a35da(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:55,005 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:55,005 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:55,005 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:A 2024-11-24T08:15:55,005 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/B, priority=13, startTime=1732436154975; duration=0sec 2024-11-24T08:15:55,005 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:55,005 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:55,005 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:B 2024-11-24T08:15:55,006 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:55,006 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/C is initiating minor compaction (all files) 2024-11-24T08:15:55,006 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/C in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:55,006 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/4e61f6442ca44f2a8bcaade410eebc99, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8082abc7ace1486badb22a7a99708b84, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/cc306666cff444c9bece8cf3b425b207] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=35.3 K 2024-11-24T08:15:55,006 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e61f6442ca44f2a8bcaade410eebc99, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1732436150558 2024-11-24T08:15:55,007 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8082abc7ace1486badb22a7a99708b84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732436151252 2024-11-24T08:15:55,007 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc306666cff444c9bece8cf3b425b207, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732436153459 2024-11-24T08:15:55,012 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#C#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:55,012 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/422665abec034d42a7189131e4c7ecda is 50, key is test_row_0/C:col10/1732436154115/Put/seqid=0 2024-11-24T08:15:55,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742277_1453 (size=12207) 2024-11-24T08:15:55,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-24T08:15:55,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:55,111 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T08:15:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:15:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:15:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:15:55,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:55,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/638aecc7f88f48a49ab558744ab3b453 is 50, key is test_row_0/A:col10/1732436154171/Put/seqid=0 2024-11-24T08:15:55,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742278_1454 (size=12001) 2024-11-24T08:15:55,120 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/638aecc7f88f48a49ab558744ab3b453 2024-11-24T08:15:55,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/aaf8ad9342734b80b99cf1416910a657 is 50, key is test_row_0/B:col10/1732436154171/Put/seqid=0 2024-11-24T08:15:55,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742279_1455 (size=12001) 2024-11-24T08:15:55,132 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/aaf8ad9342734b80b99cf1416910a657 2024-11-24T08:15:55,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/d6296180cd3c4fb2ba50a5d4fe753e8c is 50, key is test_row_0/C:col10/1732436154171/Put/seqid=0 2024-11-24T08:15:55,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742280_1456 (size=12001) 2024-11-24T08:15:55,144 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/d6296180cd3c4fb2ba50a5d4fe753e8c 2024-11-24T08:15:55,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/638aecc7f88f48a49ab558744ab3b453 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/638aecc7f88f48a49ab558744ab3b453 2024-11-24T08:15:55,151 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/638aecc7f88f48a49ab558744ab3b453, entries=150, sequenceid=104, filesize=11.7 K 2024-11-24T08:15:55,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/aaf8ad9342734b80b99cf1416910a657 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/aaf8ad9342734b80b99cf1416910a657 2024-11-24T08:15:55,157 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/aaf8ad9342734b80b99cf1416910a657, entries=150, sequenceid=104, filesize=11.7 K 2024-11-24T08:15:55,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/d6296180cd3c4fb2ba50a5d4fe753e8c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/d6296180cd3c4fb2ba50a5d4fe753e8c 2024-11-24T08:15:55,162 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/d6296180cd3c4fb2ba50a5d4fe753e8c, entries=150, sequenceid=104, filesize=11.7 K 2024-11-24T08:15:55,163 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for a55c3d95315c8337b2d7386eb1a0fc85 in 52ms, sequenceid=104, compaction requested=false 2024-11-24T08:15:55,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:55,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:55,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-24T08:15:55,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-24T08:15:55,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-24T08:15:55,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 512 msec 2024-11-24T08:15:55,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 519 msec 2024-11-24T08:15:55,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-24T08:15:55,280 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:55,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:55,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-24T08:15:55,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T08:15:55,283 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:55,283 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:55,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:55,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:55,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:15:55,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:15:55,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:55,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:15:55,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:55,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:15:55,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:55,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/f223f757a5764d8e9d29296e62a4f15b is 50, key is test_row_0/A:col10/1732436155361/Put/seqid=0 2024-11-24T08:15:55,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742281_1457 (size=16681) 2024-11-24T08:15:55,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T08:15:55,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732436155406, max timestamp=1732436155423Num uniques 859; 2024-11-24T08:15:55,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732436155406, max timestamp=1732436155423Num uniques 859; 2024-11-24T08:15:55,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T08:15:55,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T08:15:55,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732436155406, max timestamp=1732436155423Num uniques 859; 2024-11-24T08:15:55,427 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-24T08:15:55,430 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/422665abec034d42a7189131e4c7ecda as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/422665abec034d42a7189131e4c7ecda 2024-11-24T08:15:55,434 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/C of a55c3d95315c8337b2d7386eb1a0fc85 into 422665abec034d42a7189131e4c7ecda(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:55,434 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:55,434 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/C, priority=13, startTime=1732436154975; duration=0sec 2024-11-24T08:15:55,434 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:55,434 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:C 2024-11-24T08:15:55,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T08:15:55,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:55,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:55,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:55,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:55,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:55,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:55,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:55,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436215476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:55,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:55,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436215478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:55,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:55,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436215480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:55,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:55,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436215486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:55,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:55,493 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_0' in a55c3d95315c8337b2d7386eb1a0fc85 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:15:55,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:55,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436215489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:55,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T08:15:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:55,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:55,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:55,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:55,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T08:15:55,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436215600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:55,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:55,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436215600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:55,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:55,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436215600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:55,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,613 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:55,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:55,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436215610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:55,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T08:15:55,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:55,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:55,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:55,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:55,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:55,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/f223f757a5764d8e9d29296e62a4f15b 2024-11-24T08:15:55,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/990478f40abb474fbc37739f3041a804 is 50, key is test_row_0/B:col10/1732436155361/Put/seqid=0 2024-11-24T08:15:55,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742282_1458 (size=12001) 2024-11-24T08:15:55,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436215810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:55,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:55,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436215810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:55,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:55,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:55,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436215819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:55,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:55,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436215830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:55,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T08:15:55,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:55,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:55,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:55,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:55,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:55,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T08:15:56,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T08:15:56,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:56,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:56,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:56,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:56,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:56,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:56,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436216130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:56,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:56,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436216130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:56,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436216140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:56,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436216140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:56,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,146 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:56,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/990478f40abb474fbc37739f3041a804 2024-11-24T08:15:56,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/3d3ae5accef54e15b1185f9b4f43fe11 is 50, key is test_row_0/C:col10/1732436155361/Put/seqid=0 2024-11-24T08:15:56,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T08:15:56,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:56,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:56,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:56,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:56,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:56,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742283_1459 (size=12001) 2024-11-24T08:15:56,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/3d3ae5accef54e15b1185f9b4f43fe11 2024-11-24T08:15:56,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/f223f757a5764d8e9d29296e62a4f15b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/f223f757a5764d8e9d29296e62a4f15b 2024-11-24T08:15:56,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/f223f757a5764d8e9d29296e62a4f15b, entries=250, sequenceid=114, filesize=16.3 K 2024-11-24T08:15:56,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/990478f40abb474fbc37739f3041a804 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/990478f40abb474fbc37739f3041a804 2024-11-24T08:15:56,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/990478f40abb474fbc37739f3041a804, entries=150, sequenceid=114, filesize=11.7 K 2024-11-24T08:15:56,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/3d3ae5accef54e15b1185f9b4f43fe11 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3d3ae5accef54e15b1185f9b4f43fe11 2024-11-24T08:15:56,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3d3ae5accef54e15b1185f9b4f43fe11, entries=150, sequenceid=114, filesize=11.7 K 2024-11-24T08:15:56,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for a55c3d95315c8337b2d7386eb1a0fc85 in 853ms, sequenceid=114, compaction requested=true 2024-11-24T08:15:56,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:56,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:56,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:56,224 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:56,224 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:56,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:56,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:56,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:56,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:56,225 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:56,225 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:56,225 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/A is initiating minor compaction (all files) 2024-11-24T08:15:56,225 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/B is initiating minor compaction (all files) 2024-11-24T08:15:56,225 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/A in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:56,225 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/B in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:56,225 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/6401a261a34c40da997d9e0ed86f58a7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/638aecc7f88f48a49ab558744ab3b453, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/f223f757a5764d8e9d29296e62a4f15b] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=39.9 K 2024-11-24T08:15:56,225 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3edb9f22e4524ded812de8dc6b4a35da, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/aaf8ad9342734b80b99cf1416910a657, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/990478f40abb474fbc37739f3041a804] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=35.4 K 2024-11-24T08:15:56,226 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6401a261a34c40da997d9e0ed86f58a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732436153459 2024-11-24T08:15:56,226 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3edb9f22e4524ded812de8dc6b4a35da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732436153459 2024-11-24T08:15:56,226 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 638aecc7f88f48a49ab558744ab3b453, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732436154148 2024-11-24T08:15:56,226 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting aaf8ad9342734b80b99cf1416910a657, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732436154148 2024-11-24T08:15:56,226 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f223f757a5764d8e9d29296e62a4f15b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732436155361 2024-11-24T08:15:56,227 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 990478f40abb474fbc37739f3041a804, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732436155361 2024-11-24T08:15:56,234 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#A#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:56,235 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/9f5652b04f1b4d268d98fa2df0aa64b3 is 50, key is test_row_0/A:col10/1732436155361/Put/seqid=0 2024-11-24T08:15:56,238 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#B#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:56,238 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/0b0949558af947938f5477f35b45ea54 is 50, key is test_row_0/B:col10/1732436155361/Put/seqid=0 2024-11-24T08:15:56,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742284_1460 (size=12309) 2024-11-24T08:15:56,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742285_1461 (size=12309) 2024-11-24T08:15:56,252 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/9f5652b04f1b4d268d98fa2df0aa64b3 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/9f5652b04f1b4d268d98fa2df0aa64b3 2024-11-24T08:15:56,257 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/A of a55c3d95315c8337b2d7386eb1a0fc85 into 9f5652b04f1b4d268d98fa2df0aa64b3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:56,257 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:56,257 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/A, priority=13, startTime=1732436156224; duration=0sec 2024-11-24T08:15:56,257 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:56,257 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:A 2024-11-24T08:15:56,257 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:56,258 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:56,258 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/C is initiating minor compaction (all files) 2024-11-24T08:15:56,258 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/C in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:56,258 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/422665abec034d42a7189131e4c7ecda, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/d6296180cd3c4fb2ba50a5d4fe753e8c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3d3ae5accef54e15b1185f9b4f43fe11] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=35.4 K 2024-11-24T08:15:56,260 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 422665abec034d42a7189131e4c7ecda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1732436153459 2024-11-24T08:15:56,260 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6296180cd3c4fb2ba50a5d4fe753e8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732436154148 2024-11-24T08:15:56,261 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d3ae5accef54e15b1185f9b4f43fe11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732436155361 2024-11-24T08:15:56,267 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#C#compaction#386 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:56,267 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/ad7916cd0a4045228f3e0f82f88f81a7 is 50, key is test_row_0/C:col10/1732436155361/Put/seqid=0 2024-11-24T08:15:56,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742286_1462 (size=12309) 2024-11-24T08:15:56,277 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/ad7916cd0a4045228f3e0f82f88f81a7 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/ad7916cd0a4045228f3e0f82f88f81a7 2024-11-24T08:15:56,283 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/C of a55c3d95315c8337b2d7386eb1a0fc85 into ad7916cd0a4045228f3e0f82f88f81a7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:56,283 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:56,283 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/C, priority=13, startTime=1732436156224; duration=0sec 2024-11-24T08:15:56,283 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:56,283 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:C 2024-11-24T08:15:56,300 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-24T08:15:56,300 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-24T08:15:56,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-24T08:15:56,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:56,355 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-24T08:15:56,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:15:56,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:56,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:15:56,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:56,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:15:56,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:15:56,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/045c893e58184aa1b7e3e027b899ef79 is 50, key is test_row_0/A:col10/1732436155473/Put/seqid=0 2024-11-24T08:15:56,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742287_1463 (size=12151) 2024-11-24T08:15:56,363 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/045c893e58184aa1b7e3e027b899ef79 2024-11-24T08:15:56,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/c6bab56c8cb74fe19d9fe035478ae070 is 50, key is test_row_0/B:col10/1732436155473/Put/seqid=0 2024-11-24T08:15:56,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742288_1464 (size=12151) 2024-11-24T08:15:56,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T08:15:56,652 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/0b0949558af947938f5477f35b45ea54 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/0b0949558af947938f5477f35b45ea54 2024-11-24T08:15:56,656 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/B of a55c3d95315c8337b2d7386eb1a0fc85 into 0b0949558af947938f5477f35b45ea54(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:56,656 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:56,656 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/B, priority=13, startTime=1732436156224; duration=0sec 2024-11-24T08:15:56,656 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:56,656 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:B 2024-11-24T08:15:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:56,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:56,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436216662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:56,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436216673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436216673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436216673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:56,776 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/c6bab56c8cb74fe19d9fe035478ae070 2024-11-24T08:15:56,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/8581b6af43c44595abc234547ca1b245 is 50, key is test_row_0/C:col10/1732436155473/Put/seqid=0 2024-11-24T08:15:56,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742289_1465 (size=12151) 2024-11-24T08:15:56,787 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/8581b6af43c44595abc234547ca1b245 2024-11-24T08:15:56,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436216780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436216780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436216780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:56,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436216780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:56,792 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:56,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/045c893e58184aa1b7e3e027b899ef79 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/045c893e58184aa1b7e3e027b899ef79 2024-11-24T08:15:56,796 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/045c893e58184aa1b7e3e027b899ef79, entries=150, sequenceid=152, filesize=11.9 K 2024-11-24T08:15:56,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/c6bab56c8cb74fe19d9fe035478ae070 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/c6bab56c8cb74fe19d9fe035478ae070 2024-11-24T08:15:56,799 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/c6bab56c8cb74fe19d9fe035478ae070, entries=150, sequenceid=152, filesize=11.9 K 2024-11-24T08:15:56,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/8581b6af43c44595abc234547ca1b245 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8581b6af43c44595abc234547ca1b245 2024-11-24T08:15:56,805 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8581b6af43c44595abc234547ca1b245, entries=150, sequenceid=152, filesize=11.9 K 2024-11-24T08:15:56,806 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for a55c3d95315c8337b2d7386eb1a0fc85 in 451ms, sequenceid=152, compaction requested=false 2024-11-24T08:15:56,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:56,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:56,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-24T08:15:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-24T08:15:56,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-24T08:15:56,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5240 sec 2024-11-24T08:15:56,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.5290 sec 2024-11-24T08:15:57,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:57,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-24T08:15:57,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:15:57,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:57,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:15:57,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:57,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:15:57,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:57,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/90640db7341d4de19770d84283157e42 is 50, key is test_row_1/A:col10/1732436156671/Put/seqid=0 2024-11-24T08:15:57,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742290_1466 (size=14537) 2024-11-24T08:15:57,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/90640db7341d4de19770d84283157e42 2024-11-24T08:15:57,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/ec61a0294950428795c85926889f8a9b is 50, key is test_row_1/B:col10/1732436156671/Put/seqid=0 2024-11-24T08:15:57,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742291_1467 (size=9757) 2024-11-24T08:15:57,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/ec61a0294950428795c85926889f8a9b 2024-11-24T08:15:57,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/e3d5dff080f94e2c940946d4c0c1c449 is 50, key is test_row_1/C:col10/1732436156671/Put/seqid=0 2024-11-24T08:15:57,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742292_1468 (size=9757) 2024-11-24T08:15:57,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436217049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436217050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436217051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436217052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,064 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436217159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436217169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436217169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436217170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436217380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,389 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436217389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436217390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436217390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-24T08:15:57,431 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:57,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-24T08:15:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T08:15:57,433 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:57,433 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:57,434 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:57,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/e3d5dff080f94e2c940946d4c0c1c449 2024-11-24T08:15:57,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/90640db7341d4de19770d84283157e42 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/90640db7341d4de19770d84283157e42 2024-11-24T08:15:57,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/90640db7341d4de19770d84283157e42, entries=200, sequenceid=166, filesize=14.2 K 2024-11-24T08:15:57,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/ec61a0294950428795c85926889f8a9b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/ec61a0294950428795c85926889f8a9b 2024-11-24T08:15:57,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/ec61a0294950428795c85926889f8a9b, entries=100, sequenceid=166, filesize=9.5 K 2024-11-24T08:15:57,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/e3d5dff080f94e2c940946d4c0c1c449 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/e3d5dff080f94e2c940946d4c0c1c449 2024-11-24T08:15:57,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/e3d5dff080f94e2c940946d4c0c1c449, entries=100, sequenceid=166, filesize=9.5 K 2024-11-24T08:15:57,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a55c3d95315c8337b2d7386eb1a0fc85 in 463ms, sequenceid=166, compaction requested=true 2024-11-24T08:15:57,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:57,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:57,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:57,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:57,468 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:57,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:57,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:57,469 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:57,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:57,469 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38997 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:57,469 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/A is initiating minor compaction (all files) 2024-11-24T08:15:57,469 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:57,469 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/A in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:57,469 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/B is initiating minor compaction (all files) 2024-11-24T08:15:57,469 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/B in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:57,469 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/9f5652b04f1b4d268d98fa2df0aa64b3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/045c893e58184aa1b7e3e027b899ef79, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/90640db7341d4de19770d84283157e42] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=38.1 K 2024-11-24T08:15:57,470 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/0b0949558af947938f5477f35b45ea54, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/c6bab56c8cb74fe19d9fe035478ae070, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/ec61a0294950428795c85926889f8a9b] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=33.4 K 2024-11-24T08:15:57,470 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f5652b04f1b4d268d98fa2df0aa64b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732436155361 2024-11-24T08:15:57,470 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b0949558af947938f5477f35b45ea54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732436155361 2024-11-24T08:15:57,470 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 045c893e58184aa1b7e3e027b899ef79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732436155473 2024-11-24T08:15:57,470 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting c6bab56c8cb74fe19d9fe035478ae070, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732436155473 2024-11-24T08:15:57,470 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting ec61a0294950428795c85926889f8a9b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732436156671 2024-11-24T08:15:57,470 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90640db7341d4de19770d84283157e42, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732436156671 2024-11-24T08:15:57,476 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#A#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:57,477 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/22f41525ad604524875d752fe42d4145 is 50, key is test_row_0/A:col10/1732436155473/Put/seqid=0 2024-11-24T08:15:57,477 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#B#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:57,477 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/f3734f2f2b314740a2f5638108c492f8 is 50, key is test_row_0/B:col10/1732436155473/Put/seqid=0 2024-11-24T08:15:57,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742294_1470 (size=12561) 2024-11-24T08:15:57,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742293_1469 (size=12561) 2024-11-24T08:15:57,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T08:15:57,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-24T08:15:57,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:57,586 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T08:15:57,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:15:57,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:57,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:15:57,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:57,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:15:57,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:57,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/834473b97110473bb25365394916a5c1 is 50, key is test_row_0/A:col10/1732436157050/Put/seqid=0 2024-11-24T08:15:57,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742295_1471 (size=12151) 2024-11-24T08:15:57,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:57,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:57,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436217721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436217725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436217726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436217735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T08:15:57,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436217840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436217850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436217850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436217850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:57,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:57,885 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/f3734f2f2b314740a2f5638108c492f8 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/f3734f2f2b314740a2f5638108c492f8 2024-11-24T08:15:57,890 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/22f41525ad604524875d752fe42d4145 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/22f41525ad604524875d752fe42d4145 2024-11-24T08:15:57,890 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/B of a55c3d95315c8337b2d7386eb1a0fc85 into f3734f2f2b314740a2f5638108c492f8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:57,890 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:57,890 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/B, priority=13, startTime=1732436157468; duration=0sec 2024-11-24T08:15:57,890 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:57,890 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:B 2024-11-24T08:15:57,890 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:57,891 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:57,891 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/C is initiating minor compaction (all files) 2024-11-24T08:15:57,892 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/C in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:57,892 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/ad7916cd0a4045228f3e0f82f88f81a7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8581b6af43c44595abc234547ca1b245, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/e3d5dff080f94e2c940946d4c0c1c449] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=33.4 K 2024-11-24T08:15:57,892 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting ad7916cd0a4045228f3e0f82f88f81a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732436155361 2024-11-24T08:15:57,893 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 8581b6af43c44595abc234547ca1b245, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732436155473 2024-11-24T08:15:57,893 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting e3d5dff080f94e2c940946d4c0c1c449, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732436156671 2024-11-24T08:15:57,894 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/A of a55c3d95315c8337b2d7386eb1a0fc85 into 22f41525ad604524875d752fe42d4145(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:57,894 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:57,894 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/A, priority=13, startTime=1732436157468; duration=0sec 2024-11-24T08:15:57,894 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:57,894 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:A 2024-11-24T08:15:57,901 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#C#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:57,902 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/d22746a17c1a46c6a47bbae2a8261818 is 50, key is test_row_0/C:col10/1732436155473/Put/seqid=0 2024-11-24T08:15:57,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742296_1472 (size=12561) 2024-11-24T08:15:57,909 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/d22746a17c1a46c6a47bbae2a8261818 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/d22746a17c1a46c6a47bbae2a8261818 2024-11-24T08:15:57,916 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/C of a55c3d95315c8337b2d7386eb1a0fc85 into d22746a17c1a46c6a47bbae2a8261818(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:57,916 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:57,916 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/C, priority=13, startTime=1732436157469; duration=0sec 2024-11-24T08:15:57,916 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:57,916 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:C 2024-11-24T08:15:57,995 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/834473b97110473bb25365394916a5c1 2024-11-24T08:15:58,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/76ae240d43b9439595bda4ea0f31275c is 50, key is test_row_0/B:col10/1732436157050/Put/seqid=0 2024-11-24T08:15:58,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742297_1473 (size=12151) 2024-11-24T08:15:58,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T08:15:58,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436218060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:58,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436218060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:58,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:58,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436218061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,070 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:58,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436218070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:58,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436218380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:58,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436218380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:58,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436218380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:58,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436218380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:58,406 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/76ae240d43b9439595bda4ea0f31275c 2024-11-24T08:15:58,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/daf67ab103ea447c83a04aeedc2c7d38 is 50, key is test_row_0/C:col10/1732436157050/Put/seqid=0 2024-11-24T08:15:58,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742298_1474 (size=12151) 2024-11-24T08:15:58,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T08:15:58,818 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/daf67ab103ea447c83a04aeedc2c7d38 2024-11-24T08:15:58,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/834473b97110473bb25365394916a5c1 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/834473b97110473bb25365394916a5c1 2024-11-24T08:15:58,825 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/834473b97110473bb25365394916a5c1, entries=150, sequenceid=185, filesize=11.9 K 2024-11-24T08:15:58,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/76ae240d43b9439595bda4ea0f31275c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/76ae240d43b9439595bda4ea0f31275c 2024-11-24T08:15:58,829 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/76ae240d43b9439595bda4ea0f31275c, entries=150, sequenceid=185, filesize=11.9 K 2024-11-24T08:15:58,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/daf67ab103ea447c83a04aeedc2c7d38 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/daf67ab103ea447c83a04aeedc2c7d38 2024-11-24T08:15:58,833 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/daf67ab103ea447c83a04aeedc2c7d38, entries=150, sequenceid=185, filesize=11.9 K 2024-11-24T08:15:58,833 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a55c3d95315c8337b2d7386eb1a0fc85 in 1247ms, sequenceid=185, compaction requested=false 2024-11-24T08:15:58,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:58,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:58,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-24T08:15:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-24T08:15:58,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-24T08:15:58,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-11-24T08:15:58,839 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.4070 sec 2024-11-24T08:15:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:58,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T08:15:58,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:15:58,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:58,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:15:58,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:58,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:15:58,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:58,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/0b174feafd3844b2b0cd15377499effc is 50, key is test_row_0/A:col10/1732436157714/Put/seqid=0 2024-11-24T08:15:58,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742299_1475 (size=14541) 2024-11-24T08:15:58,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:58,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436218971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:58,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:58,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:58,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436218973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:58,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436218974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:58,986 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,987 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:58,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436218979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:58,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:58,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436219090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436219090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436219090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436219090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,102 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/0b174feafd3844b2b0cd15377499effc 2024-11-24T08:15:59,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/f8b28de047b348f49a4c52910b37697f is 50, key is test_row_0/B:col10/1732436157714/Put/seqid=0 2024-11-24T08:15:59,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436219310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436219310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436219310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436219310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742300_1476 (size=12151) 2024-11-24T08:15:59,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/f8b28de047b348f49a4c52910b37697f 2024-11-24T08:15:59,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/ae6dd5716a654c5ca827b79e3c9b6eec is 50, key is test_row_0/C:col10/1732436157714/Put/seqid=0 2024-11-24T08:15:59,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742301_1477 (size=12151) 2024-11-24T08:15:59,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/ae6dd5716a654c5ca827b79e3c9b6eec 2024-11-24T08:15:59,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/0b174feafd3844b2b0cd15377499effc as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/0b174feafd3844b2b0cd15377499effc 2024-11-24T08:15:59,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/0b174feafd3844b2b0cd15377499effc, entries=200, sequenceid=202, filesize=14.2 K 2024-11-24T08:15:59,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/f8b28de047b348f49a4c52910b37697f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/f8b28de047b348f49a4c52910b37697f 2024-11-24T08:15:59,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/f8b28de047b348f49a4c52910b37697f, entries=150, sequenceid=202, filesize=11.9 K 2024-11-24T08:15:59,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/ae6dd5716a654c5ca827b79e3c9b6eec as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/ae6dd5716a654c5ca827b79e3c9b6eec 2024-11-24T08:15:59,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/ae6dd5716a654c5ca827b79e3c9b6eec, entries=150, sequenceid=202, filesize=11.9 K 2024-11-24T08:15:59,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a55c3d95315c8337b2d7386eb1a0fc85 in 448ms, sequenceid=202, compaction requested=true 2024-11-24T08:15:59,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:59,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:15:59,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:59,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:15:59,352 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:59,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:59,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:15:59,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:59,352 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:59,353 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:59,353 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:59,353 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/A is initiating minor compaction (all files) 2024-11-24T08:15:59,353 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/B is initiating minor compaction (all files) 2024-11-24T08:15:59,353 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/A in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:59,353 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/B in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:59,353 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/22f41525ad604524875d752fe42d4145, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/834473b97110473bb25365394916a5c1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/0b174feafd3844b2b0cd15377499effc] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=38.3 K 2024-11-24T08:15:59,353 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/f3734f2f2b314740a2f5638108c492f8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/76ae240d43b9439595bda4ea0f31275c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/f8b28de047b348f49a4c52910b37697f] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=36.0 K 2024-11-24T08:15:59,354 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22f41525ad604524875d752fe42d4145, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732436155473 2024-11-24T08:15:59,354 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting f3734f2f2b314740a2f5638108c492f8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732436155473 2024-11-24T08:15:59,354 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 76ae240d43b9439595bda4ea0f31275c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732436157035 2024-11-24T08:15:59,354 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 834473b97110473bb25365394916a5c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732436157035 2024-11-24T08:15:59,354 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting f8b28de047b348f49a4c52910b37697f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732436157714 2024-11-24T08:15:59,354 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b174feafd3844b2b0cd15377499effc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732436157714 2024-11-24T08:15:59,361 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#B#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:59,361 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/4a48bc7df93d4201b7115597a690ce4a is 50, key is test_row_0/B:col10/1732436157714/Put/seqid=0 2024-11-24T08:15:59,364 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#A#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:59,364 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/49572591198147c292fbc2a7a4217304 is 50, key is test_row_0/A:col10/1732436157714/Put/seqid=0 2024-11-24T08:15:59,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742302_1478 (size=12663) 2024-11-24T08:15:59,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742303_1479 (size=12663) 2024-11-24T08:15:59,372 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/4a48bc7df93d4201b7115597a690ce4a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4a48bc7df93d4201b7115597a690ce4a 2024-11-24T08:15:59,373 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/49572591198147c292fbc2a7a4217304 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/49572591198147c292fbc2a7a4217304 2024-11-24T08:15:59,376 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/B of a55c3d95315c8337b2d7386eb1a0fc85 into 4a48bc7df93d4201b7115597a690ce4a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:59,376 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:59,376 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/B, priority=13, startTime=1732436159352; duration=0sec 2024-11-24T08:15:59,376 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:15:59,376 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:B 2024-11-24T08:15:59,376 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:15:59,377 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:15:59,377 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/C is initiating minor compaction (all files) 2024-11-24T08:15:59,377 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/C in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:59,377 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/d22746a17c1a46c6a47bbae2a8261818, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/daf67ab103ea447c83a04aeedc2c7d38, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/ae6dd5716a654c5ca827b79e3c9b6eec] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=36.0 K 2024-11-24T08:15:59,378 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/A of a55c3d95315c8337b2d7386eb1a0fc85 into 49572591198147c292fbc2a7a4217304(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:59,378 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:59,378 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/A, priority=13, startTime=1732436159352; duration=0sec 2024-11-24T08:15:59,378 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:59,378 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:A 2024-11-24T08:15:59,379 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting d22746a17c1a46c6a47bbae2a8261818, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732436155473 2024-11-24T08:15:59,380 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting daf67ab103ea447c83a04aeedc2c7d38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1732436157035 2024-11-24T08:15:59,380 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting ae6dd5716a654c5ca827b79e3c9b6eec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732436157714 2024-11-24T08:15:59,385 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#C#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:15:59,386 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/80ba84b554364655bbe26c3bee3bb2c9 is 50, key is test_row_0/C:col10/1732436157714/Put/seqid=0 2024-11-24T08:15:59,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742304_1480 (size=12663) 2024-11-24T08:15:59,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:15:59,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T08:15:59,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:15:59,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:59,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:15:59,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:59,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:15:59,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:15:59,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/55819fb8366d4a6fa2bbe4fc9afe2230 is 50, key is test_row_0/A:col10/1732436159501/Put/seqid=0 2024-11-24T08:15:59,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742305_1481 (size=14541) 2024-11-24T08:15:59,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/55819fb8366d4a6fa2bbe4fc9afe2230 2024-11-24T08:15:59,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/514e5fa13cca449aa87629918f008481 is 50, key is test_row_0/B:col10/1732436159501/Put/seqid=0 2024-11-24T08:15:59,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742306_1482 (size=12151) 2024-11-24T08:15:59,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436219575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-24T08:15:59,590 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:15:59,591 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:15:59,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-24T08:15:59,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T08:15:59,592 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:15:59,592 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:15:59,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:15:59,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436219630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436219630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436219630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436219630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436219689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T08:15:59,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T08:15:59,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:59,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:59,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:59,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:59,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:59,794 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/80ba84b554364655bbe26c3bee3bb2c9 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/80ba84b554364655bbe26c3bee3bb2c9 2024-11-24T08:15:59,798 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/C of a55c3d95315c8337b2d7386eb1a0fc85 into 80ba84b554364655bbe26c3bee3bb2c9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:15:59,798 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:15:59,798 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/C, priority=13, startTime=1732436159352; duration=0sec 2024-11-24T08:15:59,798 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:15:59,798 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:C 2024-11-24T08:15:59,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T08:15:59,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:59,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:15:59,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:15:59,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:59,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:15:59,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:15:59,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436219900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:15:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:15:59,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:15:59,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T08:15:59,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/514e5fa13cca449aa87629918f008481 2024-11-24T08:15:59,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/58386654285b460da668165b886ae8e6 is 50, key is test_row_0/C:col10/1732436159501/Put/seqid=0 2024-11-24T08:15:59,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742307_1483 (size=12151) 2024-11-24T08:16:00,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T08:16:00,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:00,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:00,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:00,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:00,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:00,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436220150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:00,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436220150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:00,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436220150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:00,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:00,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436220150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:00,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:00,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T08:16:00,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:00,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:00,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:00,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:00,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:00,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:00,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T08:16:00,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:00,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436220220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:00,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:00,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:00,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:00,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/58386654285b460da668165b886ae8e6 2024-11-24T08:16:00,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/55819fb8366d4a6fa2bbe4fc9afe2230 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/55819fb8366d4a6fa2bbe4fc9afe2230 2024-11-24T08:16:00,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/55819fb8366d4a6fa2bbe4fc9afe2230, entries=200, sequenceid=223, filesize=14.2 K 2024-11-24T08:16:00,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/514e5fa13cca449aa87629918f008481 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/514e5fa13cca449aa87629918f008481 2024-11-24T08:16:00,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/514e5fa13cca449aa87629918f008481, entries=150, sequenceid=223, filesize=11.9 K 2024-11-24T08:16:00,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/58386654285b460da668165b886ae8e6 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/58386654285b460da668165b886ae8e6 2024-11-24T08:16:00,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/58386654285b460da668165b886ae8e6, entries=150, sequenceid=223, filesize=11.9 K 2024-11-24T08:16:00,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a55c3d95315c8337b2d7386eb1a0fc85 in 849ms, sequenceid=223, compaction requested=false 2024-11-24T08:16:00,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:00,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-24T08:16:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:00,357 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T08:16:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:16:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:16:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:16:00,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:00,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/15fcad98304f49c983e07746231155dc is 50, key is test_row_0/A:col10/1732436159573/Put/seqid=0 2024-11-24T08:16:00,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742308_1484 (size=12151) 2024-11-24T08:16:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T08:16:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:16:00,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:00,772 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/15fcad98304f49c983e07746231155dc 2024-11-24T08:16:00,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/2ddcb2e606744edd969ec065d8706a18 is 50, key is test_row_0/B:col10/1732436159573/Put/seqid=0 2024-11-24T08:16:00,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742309_1485 (size=12151) 2024-11-24T08:16:00,787 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/2ddcb2e606744edd969ec065d8706a18 2024-11-24T08:16:00,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/f295d3bdf4de4d89ad060270de18fc46 is 50, key is test_row_0/C:col10/1732436159573/Put/seqid=0 2024-11-24T08:16:00,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742310_1486 (size=12151) 2024-11-24T08:16:00,810 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/f295d3bdf4de4d89ad060270de18fc46 2024-11-24T08:16:00,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/15fcad98304f49c983e07746231155dc as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/15fcad98304f49c983e07746231155dc 2024-11-24T08:16:00,819 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/15fcad98304f49c983e07746231155dc, entries=150, sequenceid=237, filesize=11.9 K 2024-11-24T08:16:00,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/2ddcb2e606744edd969ec065d8706a18 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/2ddcb2e606744edd969ec065d8706a18 2024-11-24T08:16:00,824 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/2ddcb2e606744edd969ec065d8706a18, entries=150, sequenceid=237, filesize=11.9 K 2024-11-24T08:16:00,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/f295d3bdf4de4d89ad060270de18fc46 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/f295d3bdf4de4d89ad060270de18fc46 2024-11-24T08:16:00,828 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/f295d3bdf4de4d89ad060270de18fc46, entries=150, sequenceid=237, filesize=11.9 K 2024-11-24T08:16:00,829 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=73.80 KB/75570 for a55c3d95315c8337b2d7386eb1a0fc85 in 472ms, sequenceid=237, compaction requested=true 2024-11-24T08:16:00,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:00,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:00,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-24T08:16:00,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-24T08:16:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:16:00,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T08:16:00,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:16:00,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:00,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:16:00,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:00,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:16:00,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:00,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-24T08:16:00,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2380 sec 2024-11-24T08:16:00,836 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.2430 sec 2024-11-24T08:16:00,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/37afda37b76e4e0dba13c030e32af66a is 50, key is test_row_0/A:col10/1732436160766/Put/seqid=0 2024-11-24T08:16:00,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742311_1487 (size=14541) 2024-11-24T08:16:00,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/37afda37b76e4e0dba13c030e32af66a 2024-11-24T08:16:00,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/653f17efd97b47fdbe016d01fe5abd13 is 50, key is test_row_0/B:col10/1732436160766/Put/seqid=0 2024-11-24T08:16:00,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742312_1488 (size=12151) 2024-11-24T08:16:00,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:00,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436220983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:00,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:01,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:01,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436221100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:01,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436221169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:01,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436221170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:01,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436221170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:01,180 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_0' in a55c3d95315c8337b2d7386eb1a0fc85 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:01,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:01,180 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in a55c3d95315c8337b2d7386eb1a0fc85 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:01,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436221170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:01,180 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in a55c3d95315c8337b2d7386eb1a0fc85 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:01,181 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in a55c3d95315c8337b2d7386eb1a0fc85 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:01,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:01,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/653f17efd97b47fdbe016d01fe5abd13 2024-11-24T08:16:01,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/0c63f16c49e84182b6c9e9cb618982b4 is 50, key is test_row_0/C:col10/1732436160766/Put/seqid=0 2024-11-24T08:16:01,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742313_1489 (size=12151) 2024-11-24T08:16:01,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:01,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436221320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:01,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:01,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436221640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:01,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:01,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/0c63f16c49e84182b6c9e9cb618982b4 2024-11-24T08:16:01,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/37afda37b76e4e0dba13c030e32af66a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/37afda37b76e4e0dba13c030e32af66a 2024-11-24T08:16:01,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/37afda37b76e4e0dba13c030e32af66a, entries=200, sequenceid=252, filesize=14.2 K 2024-11-24T08:16:01,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/653f17efd97b47fdbe016d01fe5abd13 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/653f17efd97b47fdbe016d01fe5abd13 2024-11-24T08:16:01,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/653f17efd97b47fdbe016d01fe5abd13, entries=150, sequenceid=252, filesize=11.9 K 2024-11-24T08:16:01,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/0c63f16c49e84182b6c9e9cb618982b4 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/0c63f16c49e84182b6c9e9cb618982b4 2024-11-24T08:16:01,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/0c63f16c49e84182b6c9e9cb618982b4, entries=150, sequenceid=252, filesize=11.9 K 2024-11-24T08:16:01,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a55c3d95315c8337b2d7386eb1a0fc85 in 883ms, sequenceid=252, compaction requested=true 2024-11-24T08:16:01,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:01,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:16:01,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:01,716 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:16:01,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:16:01,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:01,716 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:16:01,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:16:01,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:01,717 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:16:01,717 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53896 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:16:01,717 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/B is initiating minor compaction (all files) 2024-11-24T08:16:01,717 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/A is initiating minor compaction (all files) 2024-11-24T08:16:01,717 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/B in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:01,717 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/A in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:01,717 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4a48bc7df93d4201b7115597a690ce4a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/514e5fa13cca449aa87629918f008481, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/2ddcb2e606744edd969ec065d8706a18, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/653f17efd97b47fdbe016d01fe5abd13] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=48.0 K 2024-11-24T08:16:01,717 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/49572591198147c292fbc2a7a4217304, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/55819fb8366d4a6fa2bbe4fc9afe2230, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/15fcad98304f49c983e07746231155dc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/37afda37b76e4e0dba13c030e32af66a] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=52.6 K 2024-11-24T08:16:01,719 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a48bc7df93d4201b7115597a690ce4a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732436157714 2024-11-24T08:16:01,719 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49572591198147c292fbc2a7a4217304, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732436157714 2024-11-24T08:16:01,719 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 514e5fa13cca449aa87629918f008481, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732436158954 2024-11-24T08:16:01,719 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55819fb8366d4a6fa2bbe4fc9afe2230, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732436158954 2024-11-24T08:16:01,720 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ddcb2e606744edd969ec065d8706a18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732436159556 2024-11-24T08:16:01,720 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15fcad98304f49c983e07746231155dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732436159556 2024-11-24T08:16:01,720 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37afda37b76e4e0dba13c030e32af66a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732436160766 2024-11-24T08:16:01,720 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 653f17efd97b47fdbe016d01fe5abd13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732436160766 2024-11-24T08:16:01,726 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:01,727 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/3f243d77214c44e09bd6e285a73acfe5 is 50, key is test_row_0/B:col10/1732436160766/Put/seqid=0 2024-11-24T08:16:01,728 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#A#compaction#415 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:01,728 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/2e51fd7dfc1e4ce5bfc2fcda111d6a7c is 50, key is test_row_0/A:col10/1732436160766/Put/seqid=0 2024-11-24T08:16:01,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742315_1491 (size=12799) 2024-11-24T08:16:01,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-24T08:16:01,741 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:01,741 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:16:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-24T08:16:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T08:16:01,745 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:16:01,745 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/2e51fd7dfc1e4ce5bfc2fcda111d6a7c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/2e51fd7dfc1e4ce5bfc2fcda111d6a7c 2024-11-24T08:16:01,746 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:16:01,746 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:16:01,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742314_1490 (size=12799) 2024-11-24T08:16:01,752 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/A of a55c3d95315c8337b2d7386eb1a0fc85 into 2e51fd7dfc1e4ce5bfc2fcda111d6a7c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:01,752 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:01,752 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/A, priority=12, startTime=1732436161716; duration=0sec 2024-11-24T08:16:01,752 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:01,752 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:A 2024-11-24T08:16:01,752 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T08:16:01,754 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-24T08:16:01,754 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/C is initiating minor compaction (all files) 2024-11-24T08:16:01,755 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/C in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:01,755 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/80ba84b554364655bbe26c3bee3bb2c9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/58386654285b460da668165b886ae8e6, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/f295d3bdf4de4d89ad060270de18fc46, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/0c63f16c49e84182b6c9e9cb618982b4] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=48.0 K 2024-11-24T08:16:01,755 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80ba84b554364655bbe26c3bee3bb2c9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1732436157714 2024-11-24T08:16:01,755 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58386654285b460da668165b886ae8e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1732436158954 2024-11-24T08:16:01,756 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f295d3bdf4de4d89ad060270de18fc46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732436159556 2024-11-24T08:16:01,756 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/3f243d77214c44e09bd6e285a73acfe5 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3f243d77214c44e09bd6e285a73acfe5 2024-11-24T08:16:01,756 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c63f16c49e84182b6c9e9cb618982b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732436160766 2024-11-24T08:16:01,764 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/B of a55c3d95315c8337b2d7386eb1a0fc85 into 3f243d77214c44e09bd6e285a73acfe5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:01,764 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:01,764 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/B, priority=12, startTime=1732436161716; duration=0sec 2024-11-24T08:16:01,764 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:01,764 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:B 2024-11-24T08:16:01,772 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#C#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:01,772 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/e3d6903a192e474e9dbf6ea003543d75 is 50, key is test_row_0/C:col10/1732436160766/Put/seqid=0 2024-11-24T08:16:01,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742316_1492 (size=12799) 2024-11-24T08:16:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T08:16:01,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-24T08:16:01,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:01,899 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T08:16:01,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:16:01,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:01,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:16:01,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:01,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:16:01,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:01,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/980f7afd502d434f9db248a3bab0d930 is 50, key is test_row_0/A:col10/1732436160981/Put/seqid=0 2024-11-24T08:16:01,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742317_1493 (size=12301) 2024-11-24T08:16:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T08:16:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:16:02,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:02,185 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/e3d6903a192e474e9dbf6ea003543d75 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/e3d6903a192e474e9dbf6ea003543d75 2024-11-24T08:16:02,189 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/C of a55c3d95315c8337b2d7386eb1a0fc85 into e3d6903a192e474e9dbf6ea003543d75(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:02,189 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:02,189 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/C, priority=12, startTime=1732436161716; duration=0sec 2024-11-24T08:16:02,189 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:02,189 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:C 2024-11-24T08:16:02,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436222242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:02,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:02,308 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/980f7afd502d434f9db248a3bab0d930 2024-11-24T08:16:02,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/92119ee0e88d4097a7b94d152bb8f12e is 50, key is test_row_0/B:col10/1732436160981/Put/seqid=0 2024-11-24T08:16:02,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742318_1494 (size=12301) 2024-11-24T08:16:02,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436222360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T08:16:02,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:02,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436222569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:02,719 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/92119ee0e88d4097a7b94d152bb8f12e 2024-11-24T08:16:02,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/8ed4a92dea7243bc8e8d3c6c64afffa0 is 50, key is test_row_0/C:col10/1732436160981/Put/seqid=0 2024-11-24T08:16:02,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742319_1495 (size=12301) 2024-11-24T08:16:02,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T08:16:02,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:02,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436222890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:02,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:02,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:02,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,130 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/8ed4a92dea7243bc8e8d3c6c64afffa0 2024-11-24T08:16:03,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/980f7afd502d434f9db248a3bab0d930 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/980f7afd502d434f9db248a3bab0d930 2024-11-24T08:16:03,136 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/980f7afd502d434f9db248a3bab0d930, entries=150, sequenceid=271, filesize=12.0 K 2024-11-24T08:16:03,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/92119ee0e88d4097a7b94d152bb8f12e as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/92119ee0e88d4097a7b94d152bb8f12e 2024-11-24T08:16:03,140 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/92119ee0e88d4097a7b94d152bb8f12e, entries=150, sequenceid=271, filesize=12.0 K 2024-11-24T08:16:03,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/8ed4a92dea7243bc8e8d3c6c64afffa0 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8ed4a92dea7243bc8e8d3c6c64afffa0 2024-11-24T08:16:03,143 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8ed4a92dea7243bc8e8d3c6c64afffa0, entries=150, sequenceid=271, filesize=12.0 K 2024-11-24T08:16:03,144 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a55c3d95315c8337b2d7386eb1a0fc85 in 1245ms, sequenceid=271, compaction requested=false 2024-11-24T08:16:03,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:03,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:03,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-24T08:16:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-24T08:16:03,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-24T08:16:03,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3990 sec 2024-11-24T08:16:03,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.4060 sec 2024-11-24T08:16:03,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:16:03,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T08:16:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:16:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:16:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:16:03,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:03,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/b48728fead5244bc9f787801b7412a39 is 50, key is test_row_0/A:col10/1732436163194/Put/seqid=0 2024-11-24T08:16:03,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742320_1496 (size=14741) 2024-11-24T08:16:03,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436223225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436223228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436223237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436223239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436223350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436223350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436223359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436223359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436223410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436223570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436223570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436223571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436223571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/b48728fead5244bc9f787801b7412a39 2024-11-24T08:16:03,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/586deb54ab7c45e0a451b93aa1e6d9e4 is 50, key is test_row_0/B:col10/1732436163194/Put/seqid=0 2024-11-24T08:16:03,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742321_1497 (size=12301) 2024-11-24T08:16:03,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436223880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436223880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436223880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436223880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:03,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-24T08:16:03,900 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:03,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:16:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-24T08:16:03,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T08:16:03,902 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:16:03,903 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:16:03,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:16:04,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T08:16:04,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/586deb54ab7c45e0a451b93aa1e6d9e4 2024-11-24T08:16:04,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/40107d118e1646358c845dde626c81b5 is 50, key is test_row_0/C:col10/1732436163194/Put/seqid=0 2024-11-24T08:16:04,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742322_1498 (size=12301) 2024-11-24T08:16:04,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T08:16:04,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:04,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:04,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:04,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:04,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:04,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T08:16:04,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:04,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:04,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:04,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:04,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:04,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:04,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T08:16:04,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T08:16:04,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:04,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:04,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:04,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:04,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:04,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:04,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436224400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:04,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:04,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436224400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:04,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436224400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:04,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436224400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:04,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:04,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/40107d118e1646358c845dde626c81b5 2024-11-24T08:16:04,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/b48728fead5244bc9f787801b7412a39 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/b48728fead5244bc9f787801b7412a39 2024-11-24T08:16:04,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/b48728fead5244bc9f787801b7412a39, entries=200, sequenceid=288, filesize=14.4 K 2024-11-24T08:16:04,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436224429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:04,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/586deb54ab7c45e0a451b93aa1e6d9e4 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/586deb54ab7c45e0a451b93aa1e6d9e4 2024-11-24T08:16:04,437 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in a55c3d95315c8337b2d7386eb1a0fc85 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:04,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:04,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/586deb54ab7c45e0a451b93aa1e6d9e4, entries=150, sequenceid=288, filesize=12.0 K 2024-11-24T08:16:04,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/40107d118e1646358c845dde626c81b5 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/40107d118e1646358c845dde626c81b5 2024-11-24T08:16:04,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/40107d118e1646358c845dde626c81b5, entries=150, sequenceid=288, filesize=12.0 K 2024-11-24T08:16:04,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a55c3d95315c8337b2d7386eb1a0fc85 in 1248ms, sequenceid=288, compaction requested=true 2024-11-24T08:16:04,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:04,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:16:04,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:04,444 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:04,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:16:04,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:04,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:16:04,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:04,444 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:04,445 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39841 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:04,445 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/A is initiating minor compaction (all files) 2024-11-24T08:16:04,445 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/A in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:04,445 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/2e51fd7dfc1e4ce5bfc2fcda111d6a7c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/980f7afd502d434f9db248a3bab0d930, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/b48728fead5244bc9f787801b7412a39] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=38.9 K 2024-11-24T08:16:04,445 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:04,445 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/B is initiating minor compaction (all files) 2024-11-24T08:16:04,445 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/B in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:04,445 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3f243d77214c44e09bd6e285a73acfe5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/92119ee0e88d4097a7b94d152bb8f12e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/586deb54ab7c45e0a451b93aa1e6d9e4] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=36.5 K 2024-11-24T08:16:04,446 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e51fd7dfc1e4ce5bfc2fcda111d6a7c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732436160766 2024-11-24T08:16:04,446 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f243d77214c44e09bd6e285a73acfe5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732436160766 2024-11-24T08:16:04,446 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 92119ee0e88d4097a7b94d152bb8f12e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732436160914 2024-11-24T08:16:04,446 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 980f7afd502d434f9db248a3bab0d930, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732436160914 2024-11-24T08:16:04,446 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b48728fead5244bc9f787801b7412a39, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732436162197 2024-11-24T08:16:04,446 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 586deb54ab7c45e0a451b93aa1e6d9e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732436162197 2024-11-24T08:16:04,458 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#A#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:04,459 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/0b0c0c416ef5456fad6ff18a13ea3aeb is 50, key is test_row_0/A:col10/1732436163194/Put/seqid=0 2024-11-24T08:16:04,460 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#B#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:04,460 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/bd01ad2d48dd45f5af9ec04d0cff2f90 is 50, key is test_row_0/B:col10/1732436163194/Put/seqid=0 2024-11-24T08:16:04,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742323_1499 (size=13051) 2024-11-24T08:16:04,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742324_1500 (size=13051) 2024-11-24T08:16:04,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-24T08:16:04,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:04,514 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T08:16:04,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:16:04,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:04,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:16:04,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:04,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:16:04,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:04,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/03175f81cbfb44e3a621e136e010f133 is 50, key is test_row_0/A:col10/1732436163236/Put/seqid=0 2024-11-24T08:16:04,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742325_1501 (size=12301) 2024-11-24T08:16:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T08:16:04,798 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T08:16:04,873 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/0b0c0c416ef5456fad6ff18a13ea3aeb as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/0b0c0c416ef5456fad6ff18a13ea3aeb 2024-11-24T08:16:04,875 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/bd01ad2d48dd45f5af9ec04d0cff2f90 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/bd01ad2d48dd45f5af9ec04d0cff2f90 2024-11-24T08:16:04,879 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/A of a55c3d95315c8337b2d7386eb1a0fc85 into 0b0c0c416ef5456fad6ff18a13ea3aeb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:04,879 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/B of a55c3d95315c8337b2d7386eb1a0fc85 into bd01ad2d48dd45f5af9ec04d0cff2f90(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:04,879 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:04,879 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:04,879 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/A, priority=13, startTime=1732436164444; duration=0sec 2024-11-24T08:16:04,879 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/B, priority=13, startTime=1732436164444; duration=0sec 2024-11-24T08:16:04,879 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:04,879 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:A 2024-11-24T08:16:04,879 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:04,879 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:B 2024-11-24T08:16:04,879 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:04,880 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:04,880 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/C is initiating minor compaction (all files) 2024-11-24T08:16:04,880 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/C in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:04,880 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/e3d6903a192e474e9dbf6ea003543d75, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8ed4a92dea7243bc8e8d3c6c64afffa0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/40107d118e1646358c845dde626c81b5] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=36.5 K 2024-11-24T08:16:04,880 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3d6903a192e474e9dbf6ea003543d75, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732436160766 2024-11-24T08:16:04,881 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ed4a92dea7243bc8e8d3c6c64afffa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732436160914 2024-11-24T08:16:04,881 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40107d118e1646358c845dde626c81b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732436162197 2024-11-24T08:16:04,887 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#C#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:04,888 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/f03a51c68f9642139fcaed5a56df3e32 is 50, key is test_row_0/C:col10/1732436163194/Put/seqid=0 2024-11-24T08:16:04,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742326_1502 (size=13051) 2024-11-24T08:16:04,923 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/03175f81cbfb44e3a621e136e010f133 2024-11-24T08:16:04,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/11a1aef980a142f28a74201347b0a08d is 50, key is test_row_0/B:col10/1732436163236/Put/seqid=0 2024-11-24T08:16:04,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742327_1503 (size=12301) 2024-11-24T08:16:05,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T08:16:05,296 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/f03a51c68f9642139fcaed5a56df3e32 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/f03a51c68f9642139fcaed5a56df3e32 2024-11-24T08:16:05,300 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/C of a55c3d95315c8337b2d7386eb1a0fc85 into f03a51c68f9642139fcaed5a56df3e32(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:05,300 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:05,300 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/C, priority=13, startTime=1732436164444; duration=0sec 2024-11-24T08:16:05,300 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:05,301 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:C 2024-11-24T08:16:05,338 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/11a1aef980a142f28a74201347b0a08d 2024-11-24T08:16:05,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/3f893067906b4b04bfe380123d15a52a is 50, key is test_row_0/C:col10/1732436163236/Put/seqid=0 2024-11-24T08:16:05,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742328_1504 (size=12301) 2024-11-24T08:16:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:16:05,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:05,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436225468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436225468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436225470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436225483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436225589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436225590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436225590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436225600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,749 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/3f893067906b4b04bfe380123d15a52a 2024-11-24T08:16:05,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/03175f81cbfb44e3a621e136e010f133 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/03175f81cbfb44e3a621e136e010f133 2024-11-24T08:16:05,758 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/03175f81cbfb44e3a621e136e010f133, entries=150, sequenceid=304, filesize=12.0 K 2024-11-24T08:16:05,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/11a1aef980a142f28a74201347b0a08d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/11a1aef980a142f28a74201347b0a08d 2024-11-24T08:16:05,763 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/11a1aef980a142f28a74201347b0a08d, entries=150, sequenceid=304, filesize=12.0 K 2024-11-24T08:16:05,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/3f893067906b4b04bfe380123d15a52a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3f893067906b4b04bfe380123d15a52a 2024-11-24T08:16:05,767 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3f893067906b4b04bfe380123d15a52a, entries=150, sequenceid=304, filesize=12.0 K 2024-11-24T08:16:05,768 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for a55c3d95315c8337b2d7386eb1a0fc85 in 1255ms, sequenceid=304, compaction requested=false 2024-11-24T08:16:05,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:05,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:05,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-24T08:16:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-24T08:16:05,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-24T08:16:05,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8670 sec 2024-11-24T08:16:05,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.8720 sec 2024-11-24T08:16:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:16:05,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T08:16:05,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:16:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:16:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:16:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:05,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/4bb52c217a524fbc99f7a32a052df6f3 is 50, key is test_row_0/A:col10/1732436165438/Put/seqid=0 2024-11-24T08:16:05,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742329_1505 (size=14741) 2024-11-24T08:16:05,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436225828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436225829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436225833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436225834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436225940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436225949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436225949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436225949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-24T08:16:06,060 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:06,060 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:16:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-24T08:16:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T08:16:06,062 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:16:06,063 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:16:06,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:16:06,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:06,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436226160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:06,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T08:16:06,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:06,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436226169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:06,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:06,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436226170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436226170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:06,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/4bb52c217a524fbc99f7a32a052df6f3 2024-11-24T08:16:06,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T08:16:06,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:06,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:06,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:06,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/4f44eca7553342b4aa56c7c3317fe8aa is 50, key is test_row_0/B:col10/1732436165438/Put/seqid=0 2024-11-24T08:16:06,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742330_1506 (size=12301) 2024-11-24T08:16:06,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T08:16:06,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:06,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:06,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:06,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T08:16:06,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:06,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1732436226450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:06,459 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in a55c3d95315c8337b2d7386eb1a0fc85 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:06,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436226479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:06,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436226490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436226490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436226490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:06,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T08:16:06,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:06,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:06,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:06,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/4f44eca7553342b4aa56c7c3317fe8aa 2024-11-24T08:16:06,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/204fd226397f4d61b71571de1c5a2a6b is 50, key is test_row_0/C:col10/1732436165438/Put/seqid=0 2024-11-24T08:16:06,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742331_1507 (size=12301) 2024-11-24T08:16:06,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T08:16:06,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:06,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:06,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:06,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T08:16:06,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T08:16:06,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:06,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:06,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:06,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T08:16:06,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:06,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:06,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:06,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:07,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:07,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436226989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:07,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:07,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436227010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:07,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436227010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:07,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:07,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436227010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:07,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:07,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/204fd226397f4d61b71571de1c5a2a6b 2024-11-24T08:16:07,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/4bb52c217a524fbc99f7a32a052df6f3 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/4bb52c217a524fbc99f7a32a052df6f3 2024-11-24T08:16:07,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/4bb52c217a524fbc99f7a32a052df6f3, entries=200, sequenceid=324, filesize=14.4 K 2024-11-24T08:16:07,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/4f44eca7553342b4aa56c7c3317fe8aa as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4f44eca7553342b4aa56c7c3317fe8aa 2024-11-24T08:16:07,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4f44eca7553342b4aa56c7c3317fe8aa, entries=150, sequenceid=324, filesize=12.0 K 2024-11-24T08:16:07,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/204fd226397f4d61b71571de1c5a2a6b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/204fd226397f4d61b71571de1c5a2a6b 2024-11-24T08:16:07,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/204fd226397f4d61b71571de1c5a2a6b, entries=150, sequenceid=324, filesize=12.0 K 2024-11-24T08:16:07,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a55c3d95315c8337b2d7386eb1a0fc85 in 1245ms, sequenceid=324, compaction requested=true 2024-11-24T08:16:07,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:07,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:16:07,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:07,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:16:07,048 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:07,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:07,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:16:07,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:07,049 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:07,049 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:07,049 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:07,049 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/A is initiating minor compaction (all files) 2024-11-24T08:16:07,049 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/B is initiating minor compaction (all files) 2024-11-24T08:16:07,049 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/A in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:07,049 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/B in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:07,050 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/0b0c0c416ef5456fad6ff18a13ea3aeb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/03175f81cbfb44e3a621e136e010f133, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/4bb52c217a524fbc99f7a32a052df6f3] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=39.2 K 2024-11-24T08:16:07,050 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/bd01ad2d48dd45f5af9ec04d0cff2f90, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/11a1aef980a142f28a74201347b0a08d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4f44eca7553342b4aa56c7c3317fe8aa] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=36.8 K 2024-11-24T08:16:07,051 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b0c0c416ef5456fad6ff18a13ea3aeb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732436162197 2024-11-24T08:16:07,051 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting bd01ad2d48dd45f5af9ec04d0cff2f90, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732436162197 2024-11-24T08:16:07,051 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03175f81cbfb44e3a621e136e010f133, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732436163226 2024-11-24T08:16:07,051 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 11a1aef980a142f28a74201347b0a08d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732436163226 2024-11-24T08:16:07,052 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bb52c217a524fbc99f7a32a052df6f3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732436165438 2024-11-24T08:16:07,052 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f44eca7553342b4aa56c7c3317fe8aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732436165438 2024-11-24T08:16:07,062 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#A#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:07,063 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/a9e0985e4e5d408eb3982a07a55386c2 is 50, key is test_row_0/A:col10/1732436165438/Put/seqid=0 2024-11-24T08:16:07,063 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#B#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:07,064 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/cae4ca5cb3204b0aba498863b754a49d is 50, key is test_row_0/B:col10/1732436165438/Put/seqid=0 2024-11-24T08:16:07,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742333_1509 (size=13153) 2024-11-24T08:16:07,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742332_1508 (size=13153) 2024-11-24T08:16:07,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-24T08:16:07,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:07,134 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T08:16:07,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:16:07,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:07,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:16:07,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:07,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:16:07,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:07,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/fda2ef82210748729f1465b954611066 is 50, key is test_row_0/A:col10/1732436165831/Put/seqid=0 2024-11-24T08:16:07,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742334_1510 (size=12301) 2024-11-24T08:16:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T08:16:07,483 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/a9e0985e4e5d408eb3982a07a55386c2 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/a9e0985e4e5d408eb3982a07a55386c2 2024-11-24T08:16:07,483 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/cae4ca5cb3204b0aba498863b754a49d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/cae4ca5cb3204b0aba498863b754a49d 2024-11-24T08:16:07,487 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/B of a55c3d95315c8337b2d7386eb1a0fc85 into cae4ca5cb3204b0aba498863b754a49d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:07,487 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/A of a55c3d95315c8337b2d7386eb1a0fc85 into a9e0985e4e5d408eb3982a07a55386c2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:07,487 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:07,487 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:07,487 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/B, priority=13, startTime=1732436167048; duration=0sec 2024-11-24T08:16:07,487 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/A, priority=13, startTime=1732436167048; duration=0sec 2024-11-24T08:16:07,487 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:07,487 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:B 2024-11-24T08:16:07,487 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:07,487 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:A 2024-11-24T08:16:07,487 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:07,488 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:07,488 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/C is initiating minor compaction (all files) 2024-11-24T08:16:07,488 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/C in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:07,488 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/f03a51c68f9642139fcaed5a56df3e32, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3f893067906b4b04bfe380123d15a52a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/204fd226397f4d61b71571de1c5a2a6b] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=36.8 K 2024-11-24T08:16:07,488 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting f03a51c68f9642139fcaed5a56df3e32, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732436162197 2024-11-24T08:16:07,489 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f893067906b4b04bfe380123d15a52a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1732436163226 2024-11-24T08:16:07,489 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 204fd226397f4d61b71571de1c5a2a6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732436165438 2024-11-24T08:16:07,495 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#C#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:07,496 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/900e23afd5184711be80c43dda945aab is 50, key is test_row_0/C:col10/1732436165438/Put/seqid=0 2024-11-24T08:16:07,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742335_1511 (size=13153) 2024-11-24T08:16:07,516 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/900e23afd5184711be80c43dda945aab as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/900e23afd5184711be80c43dda945aab 2024-11-24T08:16:07,523 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/C of a55c3d95315c8337b2d7386eb1a0fc85 into 900e23afd5184711be80c43dda945aab(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:07,523 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:07,523 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/C, priority=13, startTime=1732436167048; duration=0sec 2024-11-24T08:16:07,523 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:07,523 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:C 2024-11-24T08:16:07,542 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/fda2ef82210748729f1465b954611066 2024-11-24T08:16:07,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/2103afc5378746ad854162bce5151296 is 50, key is test_row_0/B:col10/1732436165831/Put/seqid=0 2024-11-24T08:16:07,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742336_1512 (size=12301) 2024-11-24T08:16:07,570 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/2103afc5378746ad854162bce5151296 2024-11-24T08:16:07,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/6479166176f64e2cac00a8458391e7eb is 50, key is test_row_0/C:col10/1732436165831/Put/seqid=0 2024-11-24T08:16:07,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742337_1513 (size=12301) 2024-11-24T08:16:07,590 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/6479166176f64e2cac00a8458391e7eb 2024-11-24T08:16:07,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/fda2ef82210748729f1465b954611066 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/fda2ef82210748729f1465b954611066 2024-11-24T08:16:07,599 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/fda2ef82210748729f1465b954611066, entries=150, sequenceid=339, filesize=12.0 K 2024-11-24T08:16:07,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/2103afc5378746ad854162bce5151296 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/2103afc5378746ad854162bce5151296 2024-11-24T08:16:07,608 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/2103afc5378746ad854162bce5151296, entries=150, sequenceid=339, filesize=12.0 K 2024-11-24T08:16:07,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/6479166176f64e2cac00a8458391e7eb as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/6479166176f64e2cac00a8458391e7eb 2024-11-24T08:16:07,614 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/6479166176f64e2cac00a8458391e7eb, entries=150, sequenceid=339, filesize=12.0 K 2024-11-24T08:16:07,615 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for a55c3d95315c8337b2d7386eb1a0fc85 in 481ms, sequenceid=339, compaction requested=false 2024-11-24T08:16:07,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:07,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:07,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-24T08:16:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-24T08:16:07,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-24T08:16:07,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5530 sec 2024-11-24T08:16:07,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.5600 sec 2024-11-24T08:16:08,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:16:08,033 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-24T08:16:08,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:16:08,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:08,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:16:08,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:08,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:16:08,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:08,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/b3c64444bea5450a86a38f404544aba3 is 50, key is test_row_0/A:col10/1732436168018/Put/seqid=0 2024-11-24T08:16:08,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742338_1514 (size=14741) 2024-11-24T08:16:08,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/b3c64444bea5450a86a38f404544aba3 2024-11-24T08:16:08,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/51008568aa214d9da014ab321df16854 is 50, key is test_row_0/B:col10/1732436168018/Put/seqid=0 2024-11-24T08:16:08,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742339_1515 (size=12301) 2024-11-24T08:16:08,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732436168069, max timestamp=1732436168085Num uniques 859; 2024-11-24T08:16:08,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732436168069, max timestamp=1732436168085Num uniques 859; 2024-11-24T08:16:08,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T08:16:08,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T08:16:08,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732436168069, max timestamp=1732436168085Num uniques 859; 2024-11-24T08:16:08,089 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/30c28c82771d:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-24T08:16:08,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436228126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436228132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436228133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436228134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-24T08:16:08,210 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:08,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:16:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-24T08:16:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T08:16:08,213 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:16:08,213 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:16:08,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:16:08,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436228250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436228250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436228260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436228260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T08:16:08,323 INFO [Thread-1967 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1967. 2024-11-24T08:16:08,323 DEBUG [Thread-1967 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:08,323 DEBUG [Thread-1967 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:08,324 DEBUG [Thread-1967 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:08,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:08,325 INFO [Thread-1959 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1959. 2024-11-24T08:16:08,325 DEBUG [Thread-1959 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:08,325 DEBUG [Thread-1959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:08,325 DEBUG [Thread-1959 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:08,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:08,335 INFO [Thread-1963 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1963. 2024-11-24T08:16:08,335 DEBUG [Thread-1963 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:08,335 DEBUG [Thread-1963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:08,335 DEBUG [Thread-1963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:08,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:08,337 INFO [Thread-1961 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1961. 2024-11-24T08:16:08,337 DEBUG [Thread-1961 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:08,337 DEBUG [Thread-1961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:08,337 DEBUG [Thread-1961 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:08,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:08,338 INFO [Thread-1965 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1965. 2024-11-24T08:16:08,338 DEBUG [Thread-1965 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:08,338 DEBUG [Thread-1965 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:08,339 DEBUG [Thread-1965 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:08,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:08,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T08:16:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:08,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:08,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:08,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:08,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/51008568aa214d9da014ab321df16854 2024-11-24T08:16:08,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/1010cc4f5cc14e17b5f5f098ac61c9c7 is 50, key is test_row_0/C:col10/1732436168018/Put/seqid=0 2024-11-24T08:16:08,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742340_1516 (size=12301) 2024-11-24T08:16:08,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436228470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436228470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436228480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436228480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T08:16:08,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:08,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:08,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:08,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:08,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T08:16:08,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T08:16:08,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:08,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:08,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:08,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:08,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:08,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60524 deadline: 1732436228779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60520 deadline: 1732436228779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60462 deadline: 1732436228790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:08,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60454 deadline: 1732436228790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 , the old value is region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:08,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., hostname=30c28c82771d,45087,1732436016722, seqNum=2 because the exception is null or not the one we care about 2024-11-24T08:16:08,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T08:16:08,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:08,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:08,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:08,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:08,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T08:16:08,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/1010cc4f5cc14e17b5f5f098ac61c9c7 2024-11-24T08:16:08,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/b3c64444bea5450a86a38f404544aba3 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/b3c64444bea5450a86a38f404544aba3 2024-11-24T08:16:08,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/b3c64444bea5450a86a38f404544aba3, entries=200, sequenceid=352, filesize=14.4 K 2024-11-24T08:16:08,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/51008568aa214d9da014ab321df16854 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/51008568aa214d9da014ab321df16854 2024-11-24T08:16:08,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/51008568aa214d9da014ab321df16854, entries=150, sequenceid=352, filesize=12.0 K 2024-11-24T08:16:08,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/1010cc4f5cc14e17b5f5f098ac61c9c7 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/1010cc4f5cc14e17b5f5f098ac61c9c7 2024-11-24T08:16:08,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/1010cc4f5cc14e17b5f5f098ac61c9c7, entries=150, sequenceid=352, filesize=12.0 K 2024-11-24T08:16:08,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for a55c3d95315c8337b2d7386eb1a0fc85 in 849ms, sequenceid=352, compaction requested=true 2024-11-24T08:16:08,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:08,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:16:08,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:08,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:16:08,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:08,881 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:08,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a55c3d95315c8337b2d7386eb1a0fc85:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:16:08,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:08,881 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:08,882 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:08,882 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/A is initiating minor compaction (all files) 2024-11-24T08:16:08,882 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:08,882 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/A in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:08,882 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/B is initiating minor compaction (all files) 2024-11-24T08:16:08,882 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/a9e0985e4e5d408eb3982a07a55386c2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/fda2ef82210748729f1465b954611066, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/b3c64444bea5450a86a38f404544aba3] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=39.3 K 2024-11-24T08:16:08,882 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/B in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:08,882 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/cae4ca5cb3204b0aba498863b754a49d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/2103afc5378746ad854162bce5151296, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/51008568aa214d9da014ab321df16854] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=36.9 K 2024-11-24T08:16:08,882 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9e0985e4e5d408eb3982a07a55386c2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732436165438 2024-11-24T08:16:08,882 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting cae4ca5cb3204b0aba498863b754a49d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732436165438 2024-11-24T08:16:08,882 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting fda2ef82210748729f1465b954611066, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732436165828 2024-11-24T08:16:08,883 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2103afc5378746ad854162bce5151296, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732436165828 2024-11-24T08:16:08,883 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3c64444bea5450a86a38f404544aba3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732436168018 2024-11-24T08:16:08,883 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 51008568aa214d9da014ab321df16854, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732436168018 2024-11-24T08:16:08,888 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#B#compaction#441 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:08,888 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#A#compaction#442 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:08,889 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/add36c63df134fb892f939bf27597082 is 50, key is test_row_0/B:col10/1732436168018/Put/seqid=0 2024-11-24T08:16:08,889 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/7955bd4995c1421199019eb6e4b2d3f8 is 50, key is test_row_0/A:col10/1732436168018/Put/seqid=0 2024-11-24T08:16:08,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742341_1517 (size=13255) 2024-11-24T08:16:08,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742342_1518 (size=13255) 2024-11-24T08:16:08,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-24T08:16:08,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:08,977 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-24T08:16:08,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:16:08,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:16:08,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:16:08,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:16:08,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:16:08,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-24T08:16:08,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/3a39dcc5dc404a519c06a86b85a733fc is 50, key is test_row_0/A:col10/1732436168133/Put/seqid=0 2024-11-24T08:16:08,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742343_1519 (size=12301) 2024-11-24T08:16:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:16:09,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. as already flushing 2024-11-24T08:16:09,292 INFO [Thread-1954 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1954. 2024-11-24T08:16:09,292 DEBUG [Thread-1954 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:09,292 INFO [Thread-1952 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1952. 2024-11-24T08:16:09,292 DEBUG [Thread-1954 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:09,292 DEBUG [Thread-1952 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:09,292 DEBUG [Thread-1952 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:09,292 DEBUG [Thread-1954 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:09,292 DEBUG [Thread-1952 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:09,292 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:09,292 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:09,299 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/add36c63df134fb892f939bf27597082 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/add36c63df134fb892f939bf27597082 2024-11-24T08:16:09,300 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/7955bd4995c1421199019eb6e4b2d3f8 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/7955bd4995c1421199019eb6e4b2d3f8 2024-11-24T08:16:09,301 INFO [Thread-1956 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1956. 2024-11-24T08:16:09,301 DEBUG [Thread-1956 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:09,301 INFO [Thread-1950 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1950. 2024-11-24T08:16:09,301 DEBUG [Thread-1956 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:09,301 DEBUG [Thread-1950 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:09,301 DEBUG [Thread-1950 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:09,301 DEBUG [Thread-1956 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:09,301 DEBUG [Thread-1950 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:09,301 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:09,301 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:09,303 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/B of a55c3d95315c8337b2d7386eb1a0fc85 into add36c63df134fb892f939bf27597082(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:09,303 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/A of a55c3d95315c8337b2d7386eb1a0fc85 into 7955bd4995c1421199019eb6e4b2d3f8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:09,303 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:09,303 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:09,303 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/B, priority=13, startTime=1732436168881; duration=0sec 2024-11-24T08:16:09,303 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/A, priority=13, startTime=1732436168881; duration=0sec 2024-11-24T08:16:09,303 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:09,303 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:B 2024-11-24T08:16:09,303 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:09,303 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:A 2024-11-24T08:16:09,303 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:09,304 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:09,304 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): a55c3d95315c8337b2d7386eb1a0fc85/C is initiating minor compaction (all files) 2024-11-24T08:16:09,304 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a55c3d95315c8337b2d7386eb1a0fc85/C in TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:09,304 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/900e23afd5184711be80c43dda945aab, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/6479166176f64e2cac00a8458391e7eb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/1010cc4f5cc14e17b5f5f098ac61c9c7] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp, totalSize=36.9 K 2024-11-24T08:16:09,305 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 900e23afd5184711be80c43dda945aab, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732436165438 2024-11-24T08:16:09,305 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 6479166176f64e2cac00a8458391e7eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1732436165828 2024-11-24T08:16:09,305 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1010cc4f5cc14e17b5f5f098ac61c9c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732436168018 2024-11-24T08:16:09,310 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a55c3d95315c8337b2d7386eb1a0fc85#C#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:09,310 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/3f519739ee004f3682066643b6d8a229 is 50, key is test_row_0/C:col10/1732436168018/Put/seqid=0 2024-11-24T08:16:09,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742344_1520 (size=13255) 2024-11-24T08:16:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T08:16:09,385 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/3a39dcc5dc404a519c06a86b85a733fc 2024-11-24T08:16:09,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/c78d41b0c32c4e6ca1edbf758e13a27e is 50, key is test_row_0/B:col10/1732436168133/Put/seqid=0 2024-11-24T08:16:09,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742345_1521 (size=12301) 2024-11-24T08:16:09,717 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/3f519739ee004f3682066643b6d8a229 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3f519739ee004f3682066643b6d8a229 2024-11-24T08:16:09,720 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a55c3d95315c8337b2d7386eb1a0fc85/C of a55c3d95315c8337b2d7386eb1a0fc85 into 3f519739ee004f3682066643b6d8a229(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:09,720 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:09,720 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., storeName=a55c3d95315c8337b2d7386eb1a0fc85/C, priority=13, startTime=1732436168881; duration=0sec 2024-11-24T08:16:09,720 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:09,720 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a55c3d95315c8337b2d7386eb1a0fc85:C 2024-11-24T08:16:09,794 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/c78d41b0c32c4e6ca1edbf758e13a27e 2024-11-24T08:16:09,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/7bc51a9e02114cf983d28ece2f7f7cd8 is 50, key is test_row_0/C:col10/1732436168133/Put/seqid=0 2024-11-24T08:16:09,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742346_1522 (size=12301) 2024-11-24T08:16:10,203 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/7bc51a9e02114cf983d28ece2f7f7cd8 2024-11-24T08:16:10,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/3a39dcc5dc404a519c06a86b85a733fc as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/3a39dcc5dc404a519c06a86b85a733fc 2024-11-24T08:16:10,210 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/3a39dcc5dc404a519c06a86b85a733fc, entries=150, sequenceid=387, filesize=12.0 K 2024-11-24T08:16:10,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/c78d41b0c32c4e6ca1edbf758e13a27e as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/c78d41b0c32c4e6ca1edbf758e13a27e 2024-11-24T08:16:10,213 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/c78d41b0c32c4e6ca1edbf758e13a27e, entries=150, sequenceid=387, filesize=12.0 K 2024-11-24T08:16:10,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/7bc51a9e02114cf983d28ece2f7f7cd8 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/7bc51a9e02114cf983d28ece2f7f7cd8 2024-11-24T08:16:10,215 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/7bc51a9e02114cf983d28ece2f7f7cd8, entries=150, sequenceid=387, filesize=12.0 K 2024-11-24T08:16:10,216 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=26.84 KB/27480 for a55c3d95315c8337b2d7386eb1a0fc85 in 1239ms, sequenceid=387, compaction requested=false 2024-11-24T08:16:10,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for a55c3d95315c8337b2d7386eb1a0fc85: 2024-11-24T08:16:10,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:10,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-24T08:16:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-24T08:16:10,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-24T08:16:10,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0050 sec 2024-11-24T08:16:10,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 2.0090 sec 2024-11-24T08:16:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-24T08:16:10,360 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:10,501 INFO [Thread-1948 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1948. 2024-11-24T08:16:10,501 DEBUG [Thread-1948 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:10,501 DEBUG [Thread-1948 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:10,501 DEBUG [Thread-1948 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:10,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1223 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3669 rows 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1231 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3689 rows 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1235 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3702 rows 2024-11-24T08:16:10,501 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1239 2024-11-24T08:16:10,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3716 rows 2024-11-24T08:16:10,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1226 2024-11-24T08:16:10,502 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3677 rows 2024-11-24T08:16:10,502 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T08:16:10,502 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:16:10,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:10,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:10,502 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:10,503 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T08:16:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T08:16:10,505 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436170505"}]},"ts":"1732436170505"} 2024-11-24T08:16:10,506 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T08:16:10,506 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T08:16:10,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T08:16:10,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a55c3d95315c8337b2d7386eb1a0fc85, UNASSIGN}] 2024-11-24T08:16:10,509 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a55c3d95315c8337b2d7386eb1a0fc85, UNASSIGN 2024-11-24T08:16:10,509 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=a55c3d95315c8337b2d7386eb1a0fc85, regionState=CLOSING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:16:10,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a55c3d95315c8337b2d7386eb1a0fc85, UNASSIGN because future has completed 2024-11-24T08:16:10,510 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T08:16:10,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:16:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T08:16:10,662 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:16:10,663 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T08:16:10,663 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing a55c3d95315c8337b2d7386eb1a0fc85, disabling compactions & flushes 2024-11-24T08:16:10,663 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:10,663 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:10,663 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. after waiting 0 ms 2024-11-24T08:16:10,663 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:10,663 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing a55c3d95315c8337b2d7386eb1a0fc85 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T08:16:10,663 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=A 2024-11-24T08:16:10,663 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:10,663 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=B 2024-11-24T08:16:10,663 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:10,663 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a55c3d95315c8337b2d7386eb1a0fc85, store=C 2024-11-24T08:16:10,663 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:10,666 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/973574fef7c748a98869784241ae1252 is 50, key is test_row_0/A:col10/1732436169300/Put/seqid=0 2024-11-24T08:16:10,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742347_1523 (size=12301) 2024-11-24T08:16:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T08:16:11,070 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/973574fef7c748a98869784241ae1252 2024-11-24T08:16:11,075 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/ce108d1ca7a84f859668b519d7ce4c5a is 50, key is test_row_0/B:col10/1732436169300/Put/seqid=0 2024-11-24T08:16:11,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742348_1524 (size=12301) 2024-11-24T08:16:11,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T08:16:11,479 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/ce108d1ca7a84f859668b519d7ce4c5a 2024-11-24T08:16:11,487 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/3265e5807c3b43c7a3ebac25befb5725 is 50, key is test_row_0/C:col10/1732436169300/Put/seqid=0 2024-11-24T08:16:11,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742349_1525 (size=12301) 2024-11-24T08:16:11,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T08:16:11,891 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/3265e5807c3b43c7a3ebac25befb5725 2024-11-24T08:16:11,895 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/A/973574fef7c748a98869784241ae1252 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/973574fef7c748a98869784241ae1252 2024-11-24T08:16:11,897 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/973574fef7c748a98869784241ae1252, entries=150, sequenceid=398, filesize=12.0 K 2024-11-24T08:16:11,898 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/B/ce108d1ca7a84f859668b519d7ce4c5a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/ce108d1ca7a84f859668b519d7ce4c5a 2024-11-24T08:16:11,901 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/ce108d1ca7a84f859668b519d7ce4c5a, entries=150, sequenceid=398, filesize=12.0 K 2024-11-24T08:16:11,901 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/.tmp/C/3265e5807c3b43c7a3ebac25befb5725 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3265e5807c3b43c7a3ebac25befb5725 2024-11-24T08:16:11,904 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3265e5807c3b43c7a3ebac25befb5725, entries=150, sequenceid=398, filesize=12.0 K 2024-11-24T08:16:11,905 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a55c3d95315c8337b2d7386eb1a0fc85 in 1242ms, sequenceid=398, compaction requested=true 2024-11-24T08:16:11,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/93a8aabc62de4782a24a915167c4eb3b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/ebcf0c8d11ff4a149ce8521d220c9aa0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/2a75e9a702044926a2245d29442fd5ec, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/21ced72d0d734ac09ff61515da492a8e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/cf3a6db3da2e4ea29bc2bc66b1d74b32, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/6401a261a34c40da997d9e0ed86f58a7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/f88946cbb7b74a1ca884f48a555ec22a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/638aecc7f88f48a49ab558744ab3b453, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/f223f757a5764d8e9d29296e62a4f15b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/9f5652b04f1b4d268d98fa2df0aa64b3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/045c893e58184aa1b7e3e027b899ef79, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/90640db7341d4de19770d84283157e42, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/22f41525ad604524875d752fe42d4145, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/834473b97110473bb25365394916a5c1, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/0b174feafd3844b2b0cd15377499effc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/49572591198147c292fbc2a7a4217304, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/55819fb8366d4a6fa2bbe4fc9afe2230, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/15fcad98304f49c983e07746231155dc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/37afda37b76e4e0dba13c030e32af66a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/2e51fd7dfc1e4ce5bfc2fcda111d6a7c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/980f7afd502d434f9db248a3bab0d930, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/b48728fead5244bc9f787801b7412a39, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/0b0c0c416ef5456fad6ff18a13ea3aeb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/03175f81cbfb44e3a621e136e010f133, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/4bb52c217a524fbc99f7a32a052df6f3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/a9e0985e4e5d408eb3982a07a55386c2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/fda2ef82210748729f1465b954611066, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/b3c64444bea5450a86a38f404544aba3] to archive 2024-11-24T08:16:11,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:16:11,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/93a8aabc62de4782a24a915167c4eb3b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/93a8aabc62de4782a24a915167c4eb3b 2024-11-24T08:16:11,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/ebcf0c8d11ff4a149ce8521d220c9aa0 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/ebcf0c8d11ff4a149ce8521d220c9aa0 2024-11-24T08:16:11,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/2a75e9a702044926a2245d29442fd5ec to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/2a75e9a702044926a2245d29442fd5ec 2024-11-24T08:16:11,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/21ced72d0d734ac09ff61515da492a8e to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/21ced72d0d734ac09ff61515da492a8e 2024-11-24T08:16:11,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/cf3a6db3da2e4ea29bc2bc66b1d74b32 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/cf3a6db3da2e4ea29bc2bc66b1d74b32 2024-11-24T08:16:11,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/6401a261a34c40da997d9e0ed86f58a7 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/6401a261a34c40da997d9e0ed86f58a7 2024-11-24T08:16:11,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/f88946cbb7b74a1ca884f48a555ec22a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/f88946cbb7b74a1ca884f48a555ec22a 2024-11-24T08:16:11,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/638aecc7f88f48a49ab558744ab3b453 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/638aecc7f88f48a49ab558744ab3b453 2024-11-24T08:16:11,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/f223f757a5764d8e9d29296e62a4f15b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/f223f757a5764d8e9d29296e62a4f15b 2024-11-24T08:16:11,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/9f5652b04f1b4d268d98fa2df0aa64b3 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/9f5652b04f1b4d268d98fa2df0aa64b3 2024-11-24T08:16:11,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/045c893e58184aa1b7e3e027b899ef79 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/045c893e58184aa1b7e3e027b899ef79 2024-11-24T08:16:11,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/90640db7341d4de19770d84283157e42 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/90640db7341d4de19770d84283157e42 2024-11-24T08:16:11,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/22f41525ad604524875d752fe42d4145 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/22f41525ad604524875d752fe42d4145 2024-11-24T08:16:11,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/834473b97110473bb25365394916a5c1 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/834473b97110473bb25365394916a5c1 2024-11-24T08:16:11,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/0b174feafd3844b2b0cd15377499effc to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/0b174feafd3844b2b0cd15377499effc 2024-11-24T08:16:11,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/49572591198147c292fbc2a7a4217304 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/49572591198147c292fbc2a7a4217304 2024-11-24T08:16:11,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/55819fb8366d4a6fa2bbe4fc9afe2230 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/55819fb8366d4a6fa2bbe4fc9afe2230 2024-11-24T08:16:11,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/15fcad98304f49c983e07746231155dc to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/15fcad98304f49c983e07746231155dc 2024-11-24T08:16:11,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/37afda37b76e4e0dba13c030e32af66a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/37afda37b76e4e0dba13c030e32af66a 2024-11-24T08:16:11,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/2e51fd7dfc1e4ce5bfc2fcda111d6a7c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/2e51fd7dfc1e4ce5bfc2fcda111d6a7c 2024-11-24T08:16:11,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/980f7afd502d434f9db248a3bab0d930 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/980f7afd502d434f9db248a3bab0d930 2024-11-24T08:16:11,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/b48728fead5244bc9f787801b7412a39 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/b48728fead5244bc9f787801b7412a39 2024-11-24T08:16:11,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/0b0c0c416ef5456fad6ff18a13ea3aeb to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/0b0c0c416ef5456fad6ff18a13ea3aeb 2024-11-24T08:16:11,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/03175f81cbfb44e3a621e136e010f133 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/03175f81cbfb44e3a621e136e010f133 2024-11-24T08:16:11,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/4bb52c217a524fbc99f7a32a052df6f3 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/4bb52c217a524fbc99f7a32a052df6f3 2024-11-24T08:16:11,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/a9e0985e4e5d408eb3982a07a55386c2 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/a9e0985e4e5d408eb3982a07a55386c2 2024-11-24T08:16:11,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/fda2ef82210748729f1465b954611066 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/fda2ef82210748729f1465b954611066 2024-11-24T08:16:11,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/b3c64444bea5450a86a38f404544aba3 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/b3c64444bea5450a86a38f404544aba3 2024-11-24T08:16:11,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/fe21f0ba34574846bd821e2c88263c7b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/9d9c21c843ad4440bf275997d65d2556, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/0b99df2245d044b294f0394388521d11, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4523bd19e34c48d8928eeefcca38f249, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/367f916ab9e94ae2a48a701ce1f5531f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3edb9f22e4524ded812de8dc6b4a35da, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3e9f5abfb0be42299ecf2d64345997dc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/aaf8ad9342734b80b99cf1416910a657, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/0b0949558af947938f5477f35b45ea54, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/990478f40abb474fbc37739f3041a804, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/c6bab56c8cb74fe19d9fe035478ae070, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/f3734f2f2b314740a2f5638108c492f8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/ec61a0294950428795c85926889f8a9b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/76ae240d43b9439595bda4ea0f31275c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4a48bc7df93d4201b7115597a690ce4a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/f8b28de047b348f49a4c52910b37697f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/514e5fa13cca449aa87629918f008481, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/2ddcb2e606744edd969ec065d8706a18, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3f243d77214c44e09bd6e285a73acfe5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/653f17efd97b47fdbe016d01fe5abd13, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/92119ee0e88d4097a7b94d152bb8f12e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/bd01ad2d48dd45f5af9ec04d0cff2f90, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/586deb54ab7c45e0a451b93aa1e6d9e4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/11a1aef980a142f28a74201347b0a08d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/cae4ca5cb3204b0aba498863b754a49d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4f44eca7553342b4aa56c7c3317fe8aa, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/2103afc5378746ad854162bce5151296, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/51008568aa214d9da014ab321df16854] to archive 2024-11-24T08:16:11,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:16:11,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/fe21f0ba34574846bd821e2c88263c7b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/fe21f0ba34574846bd821e2c88263c7b 2024-11-24T08:16:11,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/9d9c21c843ad4440bf275997d65d2556 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/9d9c21c843ad4440bf275997d65d2556 2024-11-24T08:16:11,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/0b99df2245d044b294f0394388521d11 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/0b99df2245d044b294f0394388521d11 2024-11-24T08:16:11,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4523bd19e34c48d8928eeefcca38f249 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4523bd19e34c48d8928eeefcca38f249 2024-11-24T08:16:11,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/367f916ab9e94ae2a48a701ce1f5531f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/367f916ab9e94ae2a48a701ce1f5531f 2024-11-24T08:16:11,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3edb9f22e4524ded812de8dc6b4a35da to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3edb9f22e4524ded812de8dc6b4a35da 2024-11-24T08:16:11,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3e9f5abfb0be42299ecf2d64345997dc to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3e9f5abfb0be42299ecf2d64345997dc 2024-11-24T08:16:11,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/aaf8ad9342734b80b99cf1416910a657 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/aaf8ad9342734b80b99cf1416910a657 2024-11-24T08:16:11,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/0b0949558af947938f5477f35b45ea54 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/0b0949558af947938f5477f35b45ea54 2024-11-24T08:16:11,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/990478f40abb474fbc37739f3041a804 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/990478f40abb474fbc37739f3041a804 2024-11-24T08:16:11,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/c6bab56c8cb74fe19d9fe035478ae070 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/c6bab56c8cb74fe19d9fe035478ae070 2024-11-24T08:16:11,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/f3734f2f2b314740a2f5638108c492f8 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/f3734f2f2b314740a2f5638108c492f8 2024-11-24T08:16:11,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/ec61a0294950428795c85926889f8a9b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/ec61a0294950428795c85926889f8a9b 2024-11-24T08:16:11,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/76ae240d43b9439595bda4ea0f31275c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/76ae240d43b9439595bda4ea0f31275c 2024-11-24T08:16:11,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4a48bc7df93d4201b7115597a690ce4a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4a48bc7df93d4201b7115597a690ce4a 2024-11-24T08:16:11,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/f8b28de047b348f49a4c52910b37697f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/f8b28de047b348f49a4c52910b37697f 2024-11-24T08:16:11,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/514e5fa13cca449aa87629918f008481 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/514e5fa13cca449aa87629918f008481 2024-11-24T08:16:11,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/2ddcb2e606744edd969ec065d8706a18 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/2ddcb2e606744edd969ec065d8706a18 2024-11-24T08:16:11,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3f243d77214c44e09bd6e285a73acfe5 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/3f243d77214c44e09bd6e285a73acfe5 2024-11-24T08:16:11,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/653f17efd97b47fdbe016d01fe5abd13 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/653f17efd97b47fdbe016d01fe5abd13 2024-11-24T08:16:11,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/92119ee0e88d4097a7b94d152bb8f12e to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/92119ee0e88d4097a7b94d152bb8f12e 2024-11-24T08:16:11,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/bd01ad2d48dd45f5af9ec04d0cff2f90 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/bd01ad2d48dd45f5af9ec04d0cff2f90 2024-11-24T08:16:11,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/586deb54ab7c45e0a451b93aa1e6d9e4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/586deb54ab7c45e0a451b93aa1e6d9e4 2024-11-24T08:16:11,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/11a1aef980a142f28a74201347b0a08d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/11a1aef980a142f28a74201347b0a08d 2024-11-24T08:16:11,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/cae4ca5cb3204b0aba498863b754a49d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/cae4ca5cb3204b0aba498863b754a49d 2024-11-24T08:16:11,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4f44eca7553342b4aa56c7c3317fe8aa to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/4f44eca7553342b4aa56c7c3317fe8aa 2024-11-24T08:16:11,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/2103afc5378746ad854162bce5151296 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/2103afc5378746ad854162bce5151296 2024-11-24T08:16:11,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/51008568aa214d9da014ab321df16854 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/51008568aa214d9da014ab321df16854 2024-11-24T08:16:11,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/82289d025ede4feca0419acd36c9eecd, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/2fa5774251184cf7872960a338e11f9f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/4e61f6442ca44f2a8bcaade410eebc99, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/c6ae60adcc97407085ae08938c3f5454, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8082abc7ace1486badb22a7a99708b84, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/422665abec034d42a7189131e4c7ecda, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/cc306666cff444c9bece8cf3b425b207, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/d6296180cd3c4fb2ba50a5d4fe753e8c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/ad7916cd0a4045228f3e0f82f88f81a7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3d3ae5accef54e15b1185f9b4f43fe11, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8581b6af43c44595abc234547ca1b245, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/d22746a17c1a46c6a47bbae2a8261818, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/e3d5dff080f94e2c940946d4c0c1c449, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/daf67ab103ea447c83a04aeedc2c7d38, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/80ba84b554364655bbe26c3bee3bb2c9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/ae6dd5716a654c5ca827b79e3c9b6eec, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/58386654285b460da668165b886ae8e6, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/f295d3bdf4de4d89ad060270de18fc46, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/e3d6903a192e474e9dbf6ea003543d75, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/0c63f16c49e84182b6c9e9cb618982b4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8ed4a92dea7243bc8e8d3c6c64afffa0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/f03a51c68f9642139fcaed5a56df3e32, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/40107d118e1646358c845dde626c81b5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3f893067906b4b04bfe380123d15a52a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/900e23afd5184711be80c43dda945aab, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/204fd226397f4d61b71571de1c5a2a6b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/6479166176f64e2cac00a8458391e7eb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/1010cc4f5cc14e17b5f5f098ac61c9c7] to archive 2024-11-24T08:16:11,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:16:11,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/82289d025ede4feca0419acd36c9eecd to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/82289d025ede4feca0419acd36c9eecd 2024-11-24T08:16:11,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/2fa5774251184cf7872960a338e11f9f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/2fa5774251184cf7872960a338e11f9f 2024-11-24T08:16:11,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/4e61f6442ca44f2a8bcaade410eebc99 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/4e61f6442ca44f2a8bcaade410eebc99 2024-11-24T08:16:11,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/c6ae60adcc97407085ae08938c3f5454 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/c6ae60adcc97407085ae08938c3f5454 2024-11-24T08:16:11,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8082abc7ace1486badb22a7a99708b84 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8082abc7ace1486badb22a7a99708b84 2024-11-24T08:16:11,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/422665abec034d42a7189131e4c7ecda to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/422665abec034d42a7189131e4c7ecda 2024-11-24T08:16:11,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/cc306666cff444c9bece8cf3b425b207 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/cc306666cff444c9bece8cf3b425b207 2024-11-24T08:16:11,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/d6296180cd3c4fb2ba50a5d4fe753e8c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/d6296180cd3c4fb2ba50a5d4fe753e8c 2024-11-24T08:16:11,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/ad7916cd0a4045228f3e0f82f88f81a7 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/ad7916cd0a4045228f3e0f82f88f81a7 2024-11-24T08:16:11,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3d3ae5accef54e15b1185f9b4f43fe11 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3d3ae5accef54e15b1185f9b4f43fe11 2024-11-24T08:16:11,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8581b6af43c44595abc234547ca1b245 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8581b6af43c44595abc234547ca1b245 2024-11-24T08:16:11,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/d22746a17c1a46c6a47bbae2a8261818 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/d22746a17c1a46c6a47bbae2a8261818 2024-11-24T08:16:11,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/e3d5dff080f94e2c940946d4c0c1c449 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/e3d5dff080f94e2c940946d4c0c1c449 2024-11-24T08:16:11,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/daf67ab103ea447c83a04aeedc2c7d38 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/daf67ab103ea447c83a04aeedc2c7d38 2024-11-24T08:16:11,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/80ba84b554364655bbe26c3bee3bb2c9 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/80ba84b554364655bbe26c3bee3bb2c9 2024-11-24T08:16:11,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/ae6dd5716a654c5ca827b79e3c9b6eec to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/ae6dd5716a654c5ca827b79e3c9b6eec 2024-11-24T08:16:11,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/58386654285b460da668165b886ae8e6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/58386654285b460da668165b886ae8e6 2024-11-24T08:16:11,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/f295d3bdf4de4d89ad060270de18fc46 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/f295d3bdf4de4d89ad060270de18fc46 2024-11-24T08:16:11,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/e3d6903a192e474e9dbf6ea003543d75 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/e3d6903a192e474e9dbf6ea003543d75 2024-11-24T08:16:11,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/0c63f16c49e84182b6c9e9cb618982b4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/0c63f16c49e84182b6c9e9cb618982b4 2024-11-24T08:16:11,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8ed4a92dea7243bc8e8d3c6c64afffa0 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/8ed4a92dea7243bc8e8d3c6c64afffa0 2024-11-24T08:16:11,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/f03a51c68f9642139fcaed5a56df3e32 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/f03a51c68f9642139fcaed5a56df3e32 2024-11-24T08:16:11,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/40107d118e1646358c845dde626c81b5 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/40107d118e1646358c845dde626c81b5 2024-11-24T08:16:11,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3f893067906b4b04bfe380123d15a52a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3f893067906b4b04bfe380123d15a52a 2024-11-24T08:16:11,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/900e23afd5184711be80c43dda945aab to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/900e23afd5184711be80c43dda945aab 2024-11-24T08:16:11,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/204fd226397f4d61b71571de1c5a2a6b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/204fd226397f4d61b71571de1c5a2a6b 2024-11-24T08:16:11,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/6479166176f64e2cac00a8458391e7eb to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/6479166176f64e2cac00a8458391e7eb 2024-11-24T08:16:11,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/1010cc4f5cc14e17b5f5f098ac61c9c7 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/1010cc4f5cc14e17b5f5f098ac61c9c7 2024-11-24T08:16:11,975 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/recovered.edits/401.seqid, newMaxSeqId=401, maxSeqId=1 2024-11-24T08:16:11,975 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. 2024-11-24T08:16:11,975 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for a55c3d95315c8337b2d7386eb1a0fc85: Waiting for close lock at 1732436170663Running coprocessor pre-close hooks at 1732436170663Disabling compacts and flushes for region at 1732436170663Disabling writes for close at 1732436170663Obtaining lock to block concurrent updates at 1732436170663Preparing flush snapshotting stores in a55c3d95315c8337b2d7386eb1a0fc85 at 1732436170663Finished memstore snapshotting TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732436170663Flushing stores of TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85. at 1732436170664 (+1 ms)Flushing a55c3d95315c8337b2d7386eb1a0fc85/A: creating writer at 1732436170664Flushing a55c3d95315c8337b2d7386eb1a0fc85/A: appending metadata at 1732436170666 (+2 ms)Flushing a55c3d95315c8337b2d7386eb1a0fc85/A: closing flushed file at 1732436170666Flushing a55c3d95315c8337b2d7386eb1a0fc85/B: creating writer at 1732436171073 (+407 ms)Flushing a55c3d95315c8337b2d7386eb1a0fc85/B: appending metadata at 1732436171074 (+1 ms)Flushing a55c3d95315c8337b2d7386eb1a0fc85/B: closing flushed file at 1732436171075 (+1 ms)Flushing a55c3d95315c8337b2d7386eb1a0fc85/C: creating writer at 1732436171484 (+409 ms)Flushing a55c3d95315c8337b2d7386eb1a0fc85/C: appending metadata at 1732436171486 (+2 ms)Flushing a55c3d95315c8337b2d7386eb1a0fc85/C: closing flushed file at 1732436171486Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61bd9bf4: reopening flushed file at 1732436171894 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7acbd891: reopening flushed file at 1732436171898 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25cd4337: reopening flushed file at 1732436171901 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a55c3d95315c8337b2d7386eb1a0fc85 in 1242ms, sequenceid=398, compaction requested=true at 1732436171905 (+4 ms)Writing region close event to WAL at 1732436171973 (+68 ms)Running coprocessor post-close hooks at 1732436171975 (+2 ms)Closed at 1732436171975 2024-11-24T08:16:11,977 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:16:11,977 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=a55c3d95315c8337b2d7386eb1a0fc85, regionState=CLOSED 2024-11-24T08:16:11,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:16:11,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-24T08:16:11,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure a55c3d95315c8337b2d7386eb1a0fc85, server=30c28c82771d,45087,1732436016722 in 1.4690 sec 2024-11-24T08:16:11,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-24T08:16:11,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a55c3d95315c8337b2d7386eb1a0fc85, UNASSIGN in 1.4720 sec 2024-11-24T08:16:11,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-24T08:16:11,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4750 sec 2024-11-24T08:16:11,985 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436171984"}]},"ts":"1732436171984"} 2024-11-24T08:16:11,986 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T08:16:11,986 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T08:16:11,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4830 sec 2024-11-24T08:16:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-24T08:16:12,650 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:12,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T08:16:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:12,652 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-24T08:16:12,652 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:12,654 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:16:12,655 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/recovered.edits] 2024-11-24T08:16:12,657 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/3a39dcc5dc404a519c06a86b85a733fc to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/3a39dcc5dc404a519c06a86b85a733fc 2024-11-24T08:16:12,658 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/7955bd4995c1421199019eb6e4b2d3f8 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/7955bd4995c1421199019eb6e4b2d3f8 2024-11-24T08:16:12,659 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/973574fef7c748a98869784241ae1252 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/A/973574fef7c748a98869784241ae1252 2024-11-24T08:16:12,660 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/add36c63df134fb892f939bf27597082 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/add36c63df134fb892f939bf27597082 2024-11-24T08:16:12,661 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/c78d41b0c32c4e6ca1edbf758e13a27e to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/c78d41b0c32c4e6ca1edbf758e13a27e 2024-11-24T08:16:12,662 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/ce108d1ca7a84f859668b519d7ce4c5a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/B/ce108d1ca7a84f859668b519d7ce4c5a 2024-11-24T08:16:12,663 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3265e5807c3b43c7a3ebac25befb5725 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3265e5807c3b43c7a3ebac25befb5725 2024-11-24T08:16:12,664 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3f519739ee004f3682066643b6d8a229 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/3f519739ee004f3682066643b6d8a229 2024-11-24T08:16:12,665 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/7bc51a9e02114cf983d28ece2f7f7cd8 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/C/7bc51a9e02114cf983d28ece2f7f7cd8 2024-11-24T08:16:12,666 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/recovered.edits/401.seqid to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85/recovered.edits/401.seqid 2024-11-24T08:16:12,667 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/a55c3d95315c8337b2d7386eb1a0fc85 2024-11-24T08:16:12,667 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T08:16:12,668 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:12,670 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T08:16:12,672 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T08:16:12,672 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:12,672 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T08:16:12,673 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732436172672"}]},"ts":"9223372036854775807"} 2024-11-24T08:16:12,674 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T08:16:12,674 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a55c3d95315c8337b2d7386eb1a0fc85, NAME => 'TestAcidGuarantees,,1732436146119.a55c3d95315c8337b2d7386eb1a0fc85.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T08:16:12,674 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T08:16:12,674 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732436172674"}]},"ts":"9223372036854775807"} 2024-11-24T08:16:12,676 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T08:16:12,676 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:12,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 26 msec 2024-11-24T08:16:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-24T08:16:12,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T08:16:12,761 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:12,770 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 254), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=340 (was 294) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6242 (was 6318) 2024-11-24T08:16:12,778 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=340, ProcessCount=11, AvailableMemoryMB=6241 2024-11-24T08:16:12,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T08:16:12,779 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:16:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:12,781 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T08:16:12,781 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:12,781 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-24T08:16:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T08:16:12,782 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T08:16:12,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742350_1526 (size=1061) 2024-11-24T08:16:12,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T08:16:13,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T08:16:13,188 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6 2024-11-24T08:16:13,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742351_1527 (size=53) 2024-11-24T08:16:13,193 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:16:13,193 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing fe9967a2ec3ac6950518f327ac4152ca, disabling compactions & flushes 2024-11-24T08:16:13,193 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:13,193 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:13,193 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. after waiting 0 ms 2024-11-24T08:16:13,193 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:13,193 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:13,193 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for fe9967a2ec3ac6950518f327ac4152ca: Waiting for close lock at 1732436173193Disabling compacts and flushes for region at 1732436173193Disabling writes for close at 1732436173193Writing region close event to WAL at 1732436173193Closed at 1732436173193 2024-11-24T08:16:13,194 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T08:16:13,194 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732436173194"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732436173194"}]},"ts":"1732436173194"} 2024-11-24T08:16:13,195 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T08:16:13,196 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T08:16:13,196 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436173196"}]},"ts":"1732436173196"} 2024-11-24T08:16:13,197 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-24T08:16:13,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, ASSIGN}] 2024-11-24T08:16:13,198 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, ASSIGN 2024-11-24T08:16:13,199 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, ASSIGN; state=OFFLINE, location=30c28c82771d,45087,1732436016722; forceNewPlan=false, retain=false 2024-11-24T08:16:13,349 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fe9967a2ec3ac6950518f327ac4152ca, regionState=OPENING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:16:13,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, ASSIGN because future has completed 2024-11-24T08:16:13,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:16:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T08:16:13,505 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:13,505 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:16:13,505 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:13,505 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:16:13,505 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:13,505 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:13,506 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:13,507 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:16:13,508 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe9967a2ec3ac6950518f327ac4152ca columnFamilyName A 2024-11-24T08:16:13,508 DEBUG [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:13,508 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.HStore(327): Store=fe9967a2ec3ac6950518f327ac4152ca/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:16:13,508 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:13,509 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:16:13,509 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe9967a2ec3ac6950518f327ac4152ca columnFamilyName B 2024-11-24T08:16:13,509 DEBUG [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:13,509 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.HStore(327): Store=fe9967a2ec3ac6950518f327ac4152ca/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:16:13,509 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:13,510 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:16:13,510 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe9967a2ec3ac6950518f327ac4152ca columnFamilyName C 2024-11-24T08:16:13,510 DEBUG [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:13,510 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.HStore(327): Store=fe9967a2ec3ac6950518f327ac4152ca/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:16:13,511 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:13,511 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:13,511 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:13,511 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:13,512 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:13,512 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:13,513 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T08:16:13,514 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:13,515 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T08:16:13,515 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened fe9967a2ec3ac6950518f327ac4152ca; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70439086, jitterRate=0.04962417483329773}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T08:16:13,515 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:13,516 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for fe9967a2ec3ac6950518f327ac4152ca: Running coprocessor pre-open hook at 1732436173506Writing region info on filesystem at 1732436173506Initializing all the Stores at 1732436173506Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436173506Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436173506Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436173506Cleaning up temporary data from old regions at 1732436173512 (+6 ms)Running coprocessor post-open hooks at 1732436173515 (+3 ms)Region opened successfully at 1732436173516 (+1 ms) 2024-11-24T08:16:13,516 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., pid=148, masterSystemTime=1732436173502 2024-11-24T08:16:13,518 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:13,518 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:13,518 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=fe9967a2ec3ac6950518f327ac4152ca, regionState=OPEN, openSeqNum=2, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:16:13,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:16:13,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-24T08:16:13,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 in 169 msec 2024-11-24T08:16:13,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-24T08:16:13,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, ASSIGN in 325 msec 2024-11-24T08:16:13,524 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T08:16:13,524 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436173524"}]},"ts":"1732436173524"} 2024-11-24T08:16:13,525 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-24T08:16:13,525 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T08:16:13,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 746 msec 2024-11-24T08:16:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-24T08:16:13,920 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:13,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145ad34b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:13,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:16:13,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:16:13,922 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:16:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:16:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:16:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14035822, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:16:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:16:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:13,923 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:16:13,924 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@da29c9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:13,924 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:16:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:16:13,925 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:16:13,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:16:13,926 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T08:16:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 30c28c82771d,38469,1732436015766 2024-11-24T08:16:13,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3a9faa1c 2024-11-24T08:16:13,927 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T08:16:13,927 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T08:16:13,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-24T08:16:13,928 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T08:16:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:13,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742352_1528 (size=1097) 2024-11-24T08:16:14,338 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-24T08:16:14,338 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-24T08:16:14,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T08:16:14,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, REOPEN/MOVE}] 2024-11-24T08:16:14,343 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, REOPEN/MOVE 2024-11-24T08:16:14,343 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=fe9967a2ec3ac6950518f327ac4152ca, regionState=CLOSING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:16:14,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, REOPEN/MOVE because future has completed 2024-11-24T08:16:14,345 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T08:16:14,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:16:14,497 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,497 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T08:16:14,497 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing fe9967a2ec3ac6950518f327ac4152ca, disabling compactions & flushes 2024-11-24T08:16:14,497 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:14,497 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:14,497 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. after waiting 0 ms 2024-11-24T08:16:14,497 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:14,500 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-24T08:16:14,500 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:14,501 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for fe9967a2ec3ac6950518f327ac4152ca: Waiting for close lock at 1732436174497Running coprocessor pre-close hooks at 1732436174497Disabling compacts and flushes for region at 1732436174497Disabling writes for close at 1732436174497Writing region close event to WAL at 1732436174497Running coprocessor post-close hooks at 1732436174500 (+3 ms)Closed at 1732436174500 2024-11-24T08:16:14,501 WARN [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: fe9967a2ec3ac6950518f327ac4152ca to self. 2024-11-24T08:16:14,502 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,502 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=fe9967a2ec3ac6950518f327ac4152ca, regionState=CLOSED 2024-11-24T08:16:14,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:16:14,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-24T08:16:14,505 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 in 159 msec 2024-11-24T08:16:14,506 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, REOPEN/MOVE; state=CLOSED, location=30c28c82771d,45087,1732436016722; forceNewPlan=false, retain=true 2024-11-24T08:16:14,657 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=fe9967a2ec3ac6950518f327ac4152ca, regionState=OPENING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:16:14,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, REOPEN/MOVE because future has completed 2024-11-24T08:16:14,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:16:14,811 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:14,811 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} 2024-11-24T08:16:14,811 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,811 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T08:16:14,811 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,811 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,813 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,813 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:16:14,813 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe9967a2ec3ac6950518f327ac4152ca columnFamilyName A 2024-11-24T08:16:14,814 DEBUG [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:14,814 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.HStore(327): Store=fe9967a2ec3ac6950518f327ac4152ca/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:16:14,815 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,815 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:16:14,815 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe9967a2ec3ac6950518f327ac4152ca columnFamilyName B 2024-11-24T08:16:14,815 DEBUG [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:14,815 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.HStore(327): Store=fe9967a2ec3ac6950518f327ac4152ca/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:16:14,816 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,816 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-24T08:16:14,816 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fe9967a2ec3ac6950518f327ac4152ca columnFamilyName C 2024-11-24T08:16:14,816 DEBUG [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:14,816 INFO [StoreOpener-fe9967a2ec3ac6950518f327ac4152ca-1 {}] regionserver.HStore(327): Store=fe9967a2ec3ac6950518f327ac4152ca/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T08:16:14,816 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:14,816 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,817 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,818 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,818 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,818 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,819 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T08:16:14,820 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,820 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened fe9967a2ec3ac6950518f327ac4152ca; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72204688, jitterRate=0.07593369483947754}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T08:16:14,820 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,821 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for fe9967a2ec3ac6950518f327ac4152ca: Running coprocessor pre-open hook at 1732436174812Writing region info on filesystem at 1732436174812Initializing all the Stores at 1732436174812Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436174812Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436174812Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732436174812Cleaning up temporary data from old regions at 1732436174818 (+6 ms)Running coprocessor post-open hooks at 1732436174820 (+2 ms)Region opened successfully at 1732436174821 (+1 ms) 2024-11-24T08:16:14,821 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., pid=153, masterSystemTime=1732436174809 2024-11-24T08:16:14,823 DEBUG [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:14,823 INFO [RS_OPEN_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:14,823 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=fe9967a2ec3ac6950518f327ac4152ca, regionState=OPEN, openSeqNum=5, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:16:14,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:16:14,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-24T08:16:14,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 in 167 msec 2024-11-24T08:16:14,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-24T08:16:14,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, REOPEN/MOVE in 484 msec 2024-11-24T08:16:14,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-24T08:16:14,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-11-24T08:16:14,831 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 901 msec 2024-11-24T08:16:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-24T08:16:14,832 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:14,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143fb08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,832 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:16:14,832 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:16:14,833 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f9f3aa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:16:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:16:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:14,834 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:16:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c3a3588, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:16:14,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:16:14,836 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:16:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:16:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:16:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d1615de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:16:14,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:16:14,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:14,837 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:16:14,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56c0fb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,838 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:16:14,838 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:16:14,839 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:16:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:16:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:16:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b7b9c5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:16:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:16:14,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:14,840 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:16:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36672f9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:16:14,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:16:14,841 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a483de8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:16:14,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:14,843 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:16:14,843 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4883ba64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,843 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:16:14,843 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:16:14,844 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:16:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:16:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:16:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28e2f68c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:16:14,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:16:14,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:14,845 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:16:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64deb0fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3666b173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,846 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:16:14,846 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:16:14,847 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:16:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:16:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:16:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@527d3c67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:16:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:16:14,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:14,848 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:16:14,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@500f3aba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:16:14,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:16:14,849 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:16:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:16:14,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:16:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e746f03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:16:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:16:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:14,850 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:16:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@756f337e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,851 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:16:14,851 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:16:14,852 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:16:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:16:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:16:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25c42580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:16:14,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:16:14,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:14,853 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:16:14,854 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219278ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,854 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:16:14,854 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:16:14,855 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:16:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:16:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:16:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b6859a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:16:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:16:14,855 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:14,856 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:16:14,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4184898d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,857 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b93642c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 30c28c82771d,38469,-1 for getting cluster id 2024-11-24T08:16:14,857 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T08:16:14,857 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd039f08b-cffb-43d0-831a-6a176a1b421f' 2024-11-24T08:16:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T08:16:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d039f08b-cffb-43d0-831a-6a176a1b421f" 2024-11-24T08:16:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41877bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [30c28c82771d,38469,-1] 2024-11-24T08:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T08:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:14,858 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T08:16:14,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4af1711b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T08:16:14,859 DEBUG [Thread-2375 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:16:14,859 DEBUG [Thread-2362 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:16:14,859 DEBUG [Thread-2373 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:16:14,859 DEBUG [Thread-2366 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:16:14,859 DEBUG [Thread-2377 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:16:14,859 DEBUG [Thread-2364 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:16:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:16:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:16:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:16:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:16:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:16:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:16:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:16:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:16:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:16:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:16:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:16:14,860 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:16:14,861 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:16:14,861 DEBUG [Thread-2379 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:16:14,861 DEBUG [Thread-2368 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:16:14,861 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:16:14,861 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:16:14,861 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:16:14,861 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:16:14,861 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:16:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:16:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:16:14,862 DEBUG [Thread-2371 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:16:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:16:14,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:16:14,862 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:16:14,862 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:16:14,862 DEBUG [Thread-2360 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T08:16:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:16:14,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:16:14,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:16:14,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-24T08:16:14,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:16:14,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:16:14,864 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:16:14,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T08:16:14,865 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:16:14,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:16:14,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:16:14,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:16:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:16:14,866 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:16:14,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:16:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=30c28c82771d,45087,1732436016722, seqNum=-1] 2024-11-24T08:16:14,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T08:16:14,867 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:16:14,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:16:14,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T08:16:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:16:14,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5] 2024-11-24T08:16:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:14,875 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-24T08:16:14,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:14,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:14,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:14,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:14,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:14,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:14,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436234890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:14,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436234892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:14,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436234893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:14,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:14,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:14,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436234894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:14,899 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:14,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436234895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:14,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d89fccc3ce6f4f5690ee538b6b59edc0_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436174874/Put/seqid=0 2024-11-24T08:16:14,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742353_1529 (size=14594) 2024-11-24T08:16:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T08:16:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436235000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436235000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436235000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436235009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436235009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T08:16:15,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:15,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:15,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:15,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T08:16:15,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:15,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:15,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:15,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T08:16:15,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436235210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436235210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436235210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436235219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436235220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,313 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:15,317 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124d89fccc3ce6f4f5690ee538b6b59edc0_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d89fccc3ce6f4f5690ee538b6b59edc0_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:15,317 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/eadd5c089f5d45c2a5269edfb7c2d7aa, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:15,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/eadd5c089f5d45c2a5269edfb7c2d7aa is 175, key is test_row_0/A:col10/1732436174874/Put/seqid=0 2024-11-24T08:16:15,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742354_1530 (size=39549) 2024-11-24T08:16:15,321 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/eadd5c089f5d45c2a5269edfb7c2d7aa 2024-11-24T08:16:15,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T08:16:15,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:15,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:15,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:15,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/a93a3790727847f7a1a775af1e7aa1e9 is 50, key is test_row_0/B:col10/1732436174874/Put/seqid=0 2024-11-24T08:16:15,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742355_1531 (size=12001) 2024-11-24T08:16:15,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/a93a3790727847f7a1a775af1e7aa1e9 2024-11-24T08:16:15,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/081e8995d55c4ac58fc928528d5e65cf is 50, key is test_row_0/C:col10/1732436174874/Put/seqid=0 2024-11-24T08:16:15,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742356_1532 (size=12001) 2024-11-24T08:16:15,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T08:16:15,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:15,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:15,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:15,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T08:16:15,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436235520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436235520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436235520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436235530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436235530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:15,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:15,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T08:16:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:15,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/081e8995d55c4ac58fc928528d5e65cf 2024-11-24T08:16:15,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/eadd5c089f5d45c2a5269edfb7c2d7aa as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/eadd5c089f5d45c2a5269edfb7c2d7aa 2024-11-24T08:16:15,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T08:16:15,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:15,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:15,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:15,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:15,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/eadd5c089f5d45c2a5269edfb7c2d7aa, entries=200, sequenceid=16, filesize=38.6 K 2024-11-24T08:16:15,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/a93a3790727847f7a1a775af1e7aa1e9 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/a93a3790727847f7a1a775af1e7aa1e9 2024-11-24T08:16:15,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/a93a3790727847f7a1a775af1e7aa1e9, entries=150, sequenceid=16, filesize=11.7 K 2024-11-24T08:16:15,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/081e8995d55c4ac58fc928528d5e65cf as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/081e8995d55c4ac58fc928528d5e65cf 2024-11-24T08:16:15,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/081e8995d55c4ac58fc928528d5e65cf, entries=150, sequenceid=16, filesize=11.7 K 2024-11-24T08:16:15,794 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for fe9967a2ec3ac6950518f327ac4152ca in 919ms, sequenceid=16, compaction requested=false 2024-11-24T08:16:15,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:15,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-24T08:16:15,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:15,938 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-24T08:16:15,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:15,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:15,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:15,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:15,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:15,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:15,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124cebf9d8bc1a241c1b51edd9039d12b84_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436174892/Put/seqid=0 2024-11-24T08:16:15,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742357_1533 (size=12154) 2024-11-24T08:16:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T08:16:16,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:16,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:16,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436236040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:16,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:16,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436236040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:16,045 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,046 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:16,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436236044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:16,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436236044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436236045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:16,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436236160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:16,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436236160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:16,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,163 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:16,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436236160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:16,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:16,353 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124cebf9d8bc1a241c1b51edd9039d12b84_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cebf9d8bc1a241c1b51edd9039d12b84_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:16,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/8ed00444adc94bb4a376bdc272287be3, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:16,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/8ed00444adc94bb4a376bdc272287be3 is 175, key is test_row_0/A:col10/1732436174892/Put/seqid=0 2024-11-24T08:16:16,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742358_1534 (size=30955) 2024-11-24T08:16:16,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436236372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:16,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436236372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:16,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:16,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:16,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436236372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:16,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:16,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436236680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:16,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436236680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:16,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:16,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436236680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:16,759 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/8ed00444adc94bb4a376bdc272287be3 2024-11-24T08:16:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/c87cfd67e6f043a2bf72f5a56089f65f is 50, key is test_row_0/B:col10/1732436174892/Put/seqid=0 2024-11-24T08:16:16,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742359_1535 (size=12001) 2024-11-24T08:16:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T08:16:17,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436237060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:17,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436237060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:17,063 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in fe9967a2ec3ac6950518f327ac4152ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:17,064 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_0' in fe9967a2ec3ac6950518f327ac4152ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:17,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:17,170 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/c87cfd67e6f043a2bf72f5a56089f65f 2024-11-24T08:16:17,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/e4e84fea69524192a346d31e834c5333 is 50, key is test_row_0/C:col10/1732436174892/Put/seqid=0 2024-11-24T08:16:17,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742360_1536 (size=12001) 2024-11-24T08:16:17,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436237189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:17,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436237189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:17,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:17,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436237189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:17,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:17,496 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T08:16:17,580 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/e4e84fea69524192a346d31e834c5333 2024-11-24T08:16:17,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/8ed00444adc94bb4a376bdc272287be3 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/8ed00444adc94bb4a376bdc272287be3 2024-11-24T08:16:17,588 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/8ed00444adc94bb4a376bdc272287be3, entries=150, sequenceid=36, filesize=30.2 K 2024-11-24T08:16:17,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/c87cfd67e6f043a2bf72f5a56089f65f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/c87cfd67e6f043a2bf72f5a56089f65f 2024-11-24T08:16:17,591 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/c87cfd67e6f043a2bf72f5a56089f65f, entries=150, sequenceid=36, filesize=11.7 K 2024-11-24T08:16:17,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/e4e84fea69524192a346d31e834c5333 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/e4e84fea69524192a346d31e834c5333 2024-11-24T08:16:17,595 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/e4e84fea69524192a346d31e834c5333, entries=150, sequenceid=36, filesize=11.7 K 2024-11-24T08:16:17,596 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for fe9967a2ec3ac6950518f327ac4152ca in 1657ms, sequenceid=36, compaction requested=false 2024-11-24T08:16:17,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:17,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:17,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-24T08:16:17,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-24T08:16:17,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-24T08:16:17,599 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7310 sec 2024-11-24T08:16:17,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.7370 sec 2024-11-24T08:16:18,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:18,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-24T08:16:18,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:18,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:18,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:18,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:18,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:18,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:18,212 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124713e32ec7e814de3a05fbce219a64f99_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436176037/Put/seqid=0 2024-11-24T08:16:18,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742361_1537 (size=14594) 2024-11-24T08:16:18,216 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:18,220 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124713e32ec7e814de3a05fbce219a64f99_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124713e32ec7e814de3a05fbce219a64f99_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:18,221 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/1db5f06a5e05482a9f789ce0cd4d7ec0, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:18,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/1db5f06a5e05482a9f789ce0cd4d7ec0 is 175, key is test_row_0/A:col10/1732436176037/Put/seqid=0 2024-11-24T08:16:18,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742362_1538 (size=39549) 2024-11-24T08:16:18,234 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/1db5f06a5e05482a9f789ce0cd4d7ec0 2024-11-24T08:16:18,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/d78a440fb4314423b56972080b89aa0f is 50, key is test_row_0/B:col10/1732436176037/Put/seqid=0 2024-11-24T08:16:18,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:18,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436238238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436238238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:18,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:18,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742363_1539 (size=12001) 2024-11-24T08:16:18,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436238244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:18,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:18,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436238350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:18,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436238350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:18,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436238359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:18,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436238560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:18,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:18,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436238560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:18,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:18,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436238569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:18,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:18,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/d78a440fb4314423b56972080b89aa0f 2024-11-24T08:16:18,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/283b0ba367fa4973a20bec7170fb5910 is 50, key is test_row_0/C:col10/1732436176037/Put/seqid=0 2024-11-24T08:16:18,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742364_1540 (size=12001) 2024-11-24T08:16:18,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436238870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:18,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436238870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:18,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:18,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436238880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:18,884 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-24T08:16:19,030 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:19,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:16:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-24T08:16:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T08:16:19,033 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:16:19,033 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:16:19,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:16:19,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/283b0ba367fa4973a20bec7170fb5910 2024-11-24T08:16:19,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/1db5f06a5e05482a9f789ce0cd4d7ec0 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/1db5f06a5e05482a9f789ce0cd4d7ec0 2024-11-24T08:16:19,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/1db5f06a5e05482a9f789ce0cd4d7ec0, entries=200, sequenceid=50, filesize=38.6 K 2024-11-24T08:16:19,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/d78a440fb4314423b56972080b89aa0f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d78a440fb4314423b56972080b89aa0f 2024-11-24T08:16:19,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d78a440fb4314423b56972080b89aa0f, entries=150, sequenceid=50, filesize=11.7 K 2024-11-24T08:16:19,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/283b0ba367fa4973a20bec7170fb5910 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/283b0ba367fa4973a20bec7170fb5910 2024-11-24T08:16:19,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/283b0ba367fa4973a20bec7170fb5910, entries=150, sequenceid=50, filesize=11.7 K 2024-11-24T08:16:19,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for fe9967a2ec3ac6950518f327ac4152ca in 883ms, sequenceid=50, compaction requested=true 2024-11-24T08:16:19,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:19,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:16:19,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:19,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:16:19,085 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:19,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:19,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:16:19,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:19,086 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:19,086 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:19,086 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:19,086 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/A is initiating minor compaction (all files) 2024-11-24T08:16:19,086 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/B is initiating minor compaction (all files) 2024-11-24T08:16:19,087 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/A in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,087 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/B in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,087 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/eadd5c089f5d45c2a5269edfb7c2d7aa, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/8ed00444adc94bb4a376bdc272287be3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/1db5f06a5e05482a9f789ce0cd4d7ec0] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=107.5 K 2024-11-24T08:16:19,087 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/a93a3790727847f7a1a775af1e7aa1e9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/c87cfd67e6f043a2bf72f5a56089f65f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d78a440fb4314423b56972080b89aa0f] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=35.2 K 2024-11-24T08:16:19,087 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,087 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/eadd5c089f5d45c2a5269edfb7c2d7aa, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/8ed00444adc94bb4a376bdc272287be3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/1db5f06a5e05482a9f789ce0cd4d7ec0] 2024-11-24T08:16:19,087 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting eadd5c089f5d45c2a5269edfb7c2d7aa, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732436174864 2024-11-24T08:16:19,087 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting a93a3790727847f7a1a775af1e7aa1e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732436174864 2024-11-24T08:16:19,088 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ed00444adc94bb4a376bdc272287be3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732436174886 2024-11-24T08:16:19,088 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting c87cfd67e6f043a2bf72f5a56089f65f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732436174886 2024-11-24T08:16:19,088 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1db5f06a5e05482a9f789ce0cd4d7ec0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732436176036 2024-11-24T08:16:19,088 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting d78a440fb4314423b56972080b89aa0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732436176036 2024-11-24T08:16:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:19,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-24T08:16:19,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:19,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:19,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:19,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:19,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:19,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:19,097 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:19,099 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#B#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:19,100 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/4cae532ca33f49b1bc0703737aabffa5 is 50, key is test_row_0/B:col10/1732436176037/Put/seqid=0 2024-11-24T08:16:19,110 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112441de0a7b64e74300be9e2a8eb57bcc6c_fe9967a2ec3ac6950518f327ac4152ca store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:19,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742365_1541 (size=12104) 2024-11-24T08:16:19,112 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112441de0a7b64e74300be9e2a8eb57bcc6c_fe9967a2ec3ac6950518f327ac4152ca, store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:19,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124be9fe413d07745788f3ad1d9f2deae71_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436178234/Put/seqid=0 2024-11-24T08:16:19,112 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112441de0a7b64e74300be9e2a8eb57bcc6c_fe9967a2ec3ac6950518f327ac4152ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:19,117 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/4cae532ca33f49b1bc0703737aabffa5 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/4cae532ca33f49b1bc0703737aabffa5 2024-11-24T08:16:19,126 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/B of fe9967a2ec3ac6950518f327ac4152ca into 4cae532ca33f49b1bc0703737aabffa5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:19,126 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:19,126 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/B, priority=13, startTime=1732436179085; duration=0sec 2024-11-24T08:16:19,126 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:19,126 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:B 2024-11-24T08:16:19,126 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:19,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742366_1542 (size=14594) 2024-11-24T08:16:19,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742367_1543 (size=4469) 2024-11-24T08:16:19,127 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:19,127 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/C is initiating minor compaction (all files) 2024-11-24T08:16:19,128 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/C in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,128 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/081e8995d55c4ac58fc928528d5e65cf, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/e4e84fea69524192a346d31e834c5333, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/283b0ba367fa4973a20bec7170fb5910] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=35.2 K 2024-11-24T08:16:19,128 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 081e8995d55c4ac58fc928528d5e65cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732436174864 2024-11-24T08:16:19,129 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting e4e84fea69524192a346d31e834c5333, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1732436174886 2024-11-24T08:16:19,129 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#A#compaction#459 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:19,129 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 283b0ba367fa4973a20bec7170fb5910, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732436176036 2024-11-24T08:16:19,130 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/3ee1264a839241dc8d68f72be7469b9f is 175, key is test_row_0/A:col10/1732436176037/Put/seqid=0 2024-11-24T08:16:19,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742368_1544 (size=31058) 2024-11-24T08:16:19,138 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:19,139 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/3ee1264a839241dc8d68f72be7469b9f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/3ee1264a839241dc8d68f72be7469b9f 2024-11-24T08:16:19,139 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/cb721af4b5824f9aa96ff5d17375139d is 50, key is test_row_0/C:col10/1732436176037/Put/seqid=0 2024-11-24T08:16:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T08:16:19,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742369_1545 (size=12104) 2024-11-24T08:16:19,143 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/A of fe9967a2ec3ac6950518f327ac4152ca into 3ee1264a839241dc8d68f72be7469b9f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:19,143 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:19,143 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/A, priority=13, startTime=1732436179085; duration=0sec 2024-11-24T08:16:19,144 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:19,144 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:A 2024-11-24T08:16:19,147 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/cb721af4b5824f9aa96ff5d17375139d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/cb721af4b5824f9aa96ff5d17375139d 2024-11-24T08:16:19,151 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/C of fe9967a2ec3ac6950518f327ac4152ca into cb721af4b5824f9aa96ff5d17375139d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:19,151 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:19,151 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/C, priority=13, startTime=1732436179085; duration=0sec 2024-11-24T08:16:19,151 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:19,151 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:C 2024-11-24T08:16:19,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436239151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:19,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436239152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:19,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:19,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:19,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436239259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:19,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:19,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436239260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,263 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:19,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:19,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:19,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T08:16:19,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436239380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436239380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:19,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436239400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:19,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436239470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:19,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:19,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:19,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436239470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:19,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,473 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:19,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:19,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:19,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,527 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:19,530 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124be9fe413d07745788f3ad1d9f2deae71_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124be9fe413d07745788f3ad1d9f2deae71_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:19,531 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/8755255240054b83a7f1fb74279e416b, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:19,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/8755255240054b83a7f1fb74279e416b is 175, key is test_row_0/A:col10/1732436178234/Put/seqid=0 2024-11-24T08:16:19,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742370_1546 (size=39549) 2024-11-24T08:16:19,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:19,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:19,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T08:16:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436239780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:19,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436239780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:19,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:19,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:19,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,944 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/8755255240054b83a7f1fb74279e416b 2024-11-24T08:16:19,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:19,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:19,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:19,949 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/08bba122ee7141e59420bb4a35ebb4dc is 50, key is test_row_0/B:col10/1732436178234/Put/seqid=0 2024-11-24T08:16:19,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:19,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742371_1547 (size=12001) 2024-11-24T08:16:19,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/08bba122ee7141e59420bb4a35ebb4dc 2024-11-24T08:16:19,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/233bddec61d940738a0724d3d30644da is 50, key is test_row_0/C:col10/1732436178234/Put/seqid=0 2024-11-24T08:16:19,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742372_1548 (size=12001) 2024-11-24T08:16:20,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:20,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T08:16:20,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:20,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:20,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:20,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:20,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:20,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436240290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,292 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:20,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:20,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436240290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:20,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:20,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/233bddec61d940738a0724d3d30644da 2024-11-24T08:16:20,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/8755255240054b83a7f1fb74279e416b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/8755255240054b83a7f1fb74279e416b 2024-11-24T08:16:20,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/8755255240054b83a7f1fb74279e416b, entries=200, sequenceid=69, filesize=38.6 K 2024-11-24T08:16:20,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/08bba122ee7141e59420bb4a35ebb4dc as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/08bba122ee7141e59420bb4a35ebb4dc 2024-11-24T08:16:20,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/08bba122ee7141e59420bb4a35ebb4dc, entries=150, sequenceid=69, filesize=11.7 K 2024-11-24T08:16:20,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/233bddec61d940738a0724d3d30644da as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/233bddec61d940738a0724d3d30644da 2024-11-24T08:16:20,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/233bddec61d940738a0724d3d30644da, entries=150, sequenceid=69, filesize=11.7 K 2024-11-24T08:16:20,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for fe9967a2ec3ac6950518f327ac4152ca in 1285ms, sequenceid=69, compaction requested=false 2024-11-24T08:16:20,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:20,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:20,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-24T08:16:20,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:20,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:20,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:20,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:20,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:20,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:20,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124263f96985bd44e188649ed90573e64ca_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436180391/Put/seqid=0 2024-11-24T08:16:20,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742373_1549 (size=14594) 2024-11-24T08:16:20,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:20,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:20,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:20,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:20,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436240423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436240424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:20,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436240426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:20,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:20,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436240540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:20,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436240540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436240540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:20,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:20,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:20,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:20,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:20,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:20,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:20,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:20,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:20,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436240749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436240750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:20,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436240750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:20,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:20,802 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:20,805 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124263f96985bd44e188649ed90573e64ca_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124263f96985bd44e188649ed90573e64ca_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:20,806 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/75f2d7bb8ec04c50bcb6a4d89ec88aee, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:20,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/75f2d7bb8ec04c50bcb6a4d89ec88aee is 175, key is test_row_0/A:col10/1732436180391/Put/seqid=0 2024-11-24T08:16:20,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742374_1550 (size=39549) 2024-11-24T08:16:20,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:20,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:20,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:20,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:20,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:20,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:21,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:21,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:21,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:21,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436241060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:21,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:21,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436241060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:21,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436241060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:21,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:21,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T08:16:21,216 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/75f2d7bb8ec04c50bcb6a4d89ec88aee 2024-11-24T08:16:21,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/ee786f8951dc40388274eab8cdaf35af is 50, key is test_row_0/B:col10/1732436180391/Put/seqid=0 2024-11-24T08:16:21,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742375_1551 (size=12001) 2024-11-24T08:16:21,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436241310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:21,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436241310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:21,314 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in fe9967a2ec3ac6950518f327ac4152ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:21,314 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,314 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in fe9967a2ec3ac6950518f327ac4152ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:21,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:21,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:21,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:21,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:21,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:21,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:21,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:21,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:21,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436241570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436241570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:21,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436241570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:21,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:21,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:21,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:21,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/ee786f8951dc40388274eab8cdaf35af 2024-11-24T08:16:21,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:21,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/d175b6b401a146c293484e9248bbd409 is 50, key is test_row_0/C:col10/1732436180391/Put/seqid=0 2024-11-24T08:16:21,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742376_1552 (size=12001) 2024-11-24T08:16:21,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:21,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:21,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:21,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:21,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:21,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:21,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:21,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:21,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:21,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:22,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/d175b6b401a146c293484e9248bbd409 2024-11-24T08:16:22,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/75f2d7bb8ec04c50bcb6a4d89ec88aee as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/75f2d7bb8ec04c50bcb6a4d89ec88aee 2024-11-24T08:16:22,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/75f2d7bb8ec04c50bcb6a4d89ec88aee, entries=200, sequenceid=86, filesize=38.6 K 2024-11-24T08:16:22,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/ee786f8951dc40388274eab8cdaf35af as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/ee786f8951dc40388274eab8cdaf35af 2024-11-24T08:16:22,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/ee786f8951dc40388274eab8cdaf35af, entries=150, sequenceid=86, filesize=11.7 K 2024-11-24T08:16:22,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/d175b6b401a146c293484e9248bbd409 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/d175b6b401a146c293484e9248bbd409 2024-11-24T08:16:22,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/d175b6b401a146c293484e9248bbd409, entries=150, sequenceid=86, filesize=11.7 K 2024-11-24T08:16:22,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for fe9967a2ec3ac6950518f327ac4152ca in 1664ms, sequenceid=86, compaction requested=true 2024-11-24T08:16:22,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:22,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:16:22,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:22,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:16:22,057 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:22,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:22,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:16:22,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:22,057 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:22,057 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110156 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:22,057 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:22,057 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/B is initiating minor compaction (all files) 2024-11-24T08:16:22,057 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/A is initiating minor compaction (all files) 2024-11-24T08:16:22,057 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/A in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:22,057 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/B in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:22,057 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/3ee1264a839241dc8d68f72be7469b9f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/8755255240054b83a7f1fb74279e416b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/75f2d7bb8ec04c50bcb6a4d89ec88aee] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=107.6 K 2024-11-24T08:16:22,057 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:22,057 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/4cae532ca33f49b1bc0703737aabffa5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/08bba122ee7141e59420bb4a35ebb4dc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/ee786f8951dc40388274eab8cdaf35af] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=35.3 K 2024-11-24T08:16:22,057 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/3ee1264a839241dc8d68f72be7469b9f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/8755255240054b83a7f1fb74279e416b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/75f2d7bb8ec04c50bcb6a4d89ec88aee] 2024-11-24T08:16:22,058 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ee1264a839241dc8d68f72be7469b9f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732436176036 2024-11-24T08:16:22,058 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cae532ca33f49b1bc0703737aabffa5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732436176036 2024-11-24T08:16:22,058 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8755255240054b83a7f1fb74279e416b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732436178226 2024-11-24T08:16:22,058 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 08bba122ee7141e59420bb4a35ebb4dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732436178226 2024-11-24T08:16:22,058 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75f2d7bb8ec04c50bcb6a4d89ec88aee, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732436179142 2024-11-24T08:16:22,058 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting ee786f8951dc40388274eab8cdaf35af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732436179142 2024-11-24T08:16:22,068 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:22,069 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#B#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:22,069 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/f255dea9a5184ac0b30df7b6733812fb is 50, key is test_row_0/B:col10/1732436180391/Put/seqid=0 2024-11-24T08:16:22,069 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411246d949eaaaff84b1f93777dfbf6e3ede3_fe9967a2ec3ac6950518f327ac4152ca store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:22,072 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411246d949eaaaff84b1f93777dfbf6e3ede3_fe9967a2ec3ac6950518f327ac4152ca, store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:22,072 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411246d949eaaaff84b1f93777dfbf6e3ede3_fe9967a2ec3ac6950518f327ac4152ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:22,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742378_1554 (size=4469) 2024-11-24T08:16:22,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742377_1553 (size=12207) 2024-11-24T08:16:22,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-24T08:16:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:22,090 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T08:16:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:22,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:22,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112411cc15afbb4f4103907cec38b9f823fa_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436180416/Put/seqid=0 2024-11-24T08:16:22,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742379_1555 (size=12154) 2024-11-24T08:16:22,476 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#A#compaction#469 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:22,477 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/677df9f3fd46436eaa40986ccaa2b6c8 is 175, key is test_row_0/A:col10/1732436180391/Put/seqid=0 2024-11-24T08:16:22,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742380_1556 (size=31161) 2024-11-24T08:16:22,483 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/f255dea9a5184ac0b30df7b6733812fb as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/f255dea9a5184ac0b30df7b6733812fb 2024-11-24T08:16:22,487 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/677df9f3fd46436eaa40986ccaa2b6c8 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/677df9f3fd46436eaa40986ccaa2b6c8 2024-11-24T08:16:22,489 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/B of fe9967a2ec3ac6950518f327ac4152ca into f255dea9a5184ac0b30df7b6733812fb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:22,489 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:22,489 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/B, priority=13, startTime=1732436182056; duration=0sec 2024-11-24T08:16:22,489 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:22,489 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:B 2024-11-24T08:16:22,489 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:22,492 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:22,492 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/C is initiating minor compaction (all files) 2024-11-24T08:16:22,492 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/C in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:22,492 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/cb721af4b5824f9aa96ff5d17375139d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/233bddec61d940738a0724d3d30644da, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/d175b6b401a146c293484e9248bbd409] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=35.3 K 2024-11-24T08:16:22,492 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/A of fe9967a2ec3ac6950518f327ac4152ca into 677df9f3fd46436eaa40986ccaa2b6c8(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:22,492 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:22,492 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/A, priority=13, startTime=1732436182056; duration=0sec 2024-11-24T08:16:22,492 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:22,492 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:A 2024-11-24T08:16:22,492 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting cb721af4b5824f9aa96ff5d17375139d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732436176036 2024-11-24T08:16:22,493 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 233bddec61d940738a0724d3d30644da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732436178226 2024-11-24T08:16:22,493 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting d175b6b401a146c293484e9248bbd409, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732436179142 2024-11-24T08:16:22,499 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:22,499 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/eb488fa369564abd99d14fd144d14b93 is 50, key is test_row_0/C:col10/1732436180391/Put/seqid=0 2024-11-24T08:16:22,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:22,503 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112411cc15afbb4f4103907cec38b9f823fa_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112411cc15afbb4f4103907cec38b9f823fa_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:22,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/cc64a699a1d74970a7ba2e7670e8838c, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:22,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/cc64a699a1d74970a7ba2e7670e8838c is 175, key is test_row_0/A:col10/1732436180416/Put/seqid=0 2024-11-24T08:16:22,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742381_1557 (size=12207) 2024-11-24T08:16:22,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742382_1558 (size=30955) 2024-11-24T08:16:22,509 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/cc64a699a1d74970a7ba2e7670e8838c 2024-11-24T08:16:22,511 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/eb488fa369564abd99d14fd144d14b93 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/eb488fa369564abd99d14fd144d14b93 2024-11-24T08:16:22,515 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/C of fe9967a2ec3ac6950518f327ac4152ca into eb488fa369564abd99d14fd144d14b93(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:22,515 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:22,515 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/C, priority=13, startTime=1732436182057; duration=0sec 2024-11-24T08:16:22,515 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:22,515 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:C 2024-11-24T08:16:22,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/d0caff9623754bb48a85f72d025facac is 50, key is test_row_0/B:col10/1732436180416/Put/seqid=0 2024-11-24T08:16:22,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742383_1559 (size=12001) 2024-11-24T08:16:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:22,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:22,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436242602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:22,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436242603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:22,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436242605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:22,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436242709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:22,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436242710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:22,713 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,714 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:22,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436242723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:22,921 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/d0caff9623754bb48a85f72d025facac 2024-11-24T08:16:22,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:22,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436242920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436242920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:22,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/294b216ba71742ce96f606746f71fc2b is 50, key is test_row_0/C:col10/1732436180416/Put/seqid=0 2024-11-24T08:16:22,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436242929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:22,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:22,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:22,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742384_1560 (size=12001) 2024-11-24T08:16:23,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T08:16:23,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436243230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:23,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436243230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:23,233 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:23,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:23,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:23,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436243240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:23,331 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/294b216ba71742ce96f606746f71fc2b 2024-11-24T08:16:23,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436243330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:23,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:23,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436243330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:23,334 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in fe9967a2ec3ac6950518f327ac4152ca of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:23,336 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_2' in fe9967a2ec3ac6950518f327ac4152ca of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:23,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/cc64a699a1d74970a7ba2e7670e8838c as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/cc64a699a1d74970a7ba2e7670e8838c 2024-11-24T08:16:23,341 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/cc64a699a1d74970a7ba2e7670e8838c, entries=150, sequenceid=104, filesize=30.2 K 2024-11-24T08:16:23,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/d0caff9623754bb48a85f72d025facac as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d0caff9623754bb48a85f72d025facac 2024-11-24T08:16:23,344 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d0caff9623754bb48a85f72d025facac, entries=150, sequenceid=104, filesize=11.7 K 2024-11-24T08:16:23,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/294b216ba71742ce96f606746f71fc2b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/294b216ba71742ce96f606746f71fc2b 2024-11-24T08:16:23,348 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/294b216ba71742ce96f606746f71fc2b, entries=150, sequenceid=104, filesize=11.7 K 2024-11-24T08:16:23,349 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for fe9967a2ec3ac6950518f327ac4152ca in 1259ms, sequenceid=104, compaction requested=false 2024-11-24T08:16:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:23,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-24T08:16:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-24T08:16:23,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-24T08:16:23,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.3170 sec 2024-11-24T08:16:23,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 4.3220 sec 2024-11-24T08:16:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:23,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T08:16:23,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:23,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:23,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:23,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:23,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:23,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:23,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242991a133f32f45f6afae35b8b4d78184_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436183742/Put/seqid=0 2024-11-24T08:16:23,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742385_1561 (size=17034) 2024-11-24T08:16:23,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436243767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:23,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436243768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:23,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436243772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:23,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436243879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:23,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:23,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436243879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:23,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436243880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:24,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436244090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:24,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436244090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:24,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436244090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:24,155 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:24,157 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411242991a133f32f45f6afae35b8b4d78184_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242991a133f32f45f6afae35b8b4d78184_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:24,158 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/2d7ecd8406b54030bbc4f4842cda4e54, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:24,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/2d7ecd8406b54030bbc4f4842cda4e54 is 175, key is test_row_0/A:col10/1732436183742/Put/seqid=0 2024-11-24T08:16:24,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742386_1562 (size=48139) 2024-11-24T08:16:24,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436244400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:24,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:24,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436244400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:24,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:24,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436244400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:24,563 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=123, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/2d7ecd8406b54030bbc4f4842cda4e54 2024-11-24T08:16:24,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/544632a2ea34479aa3376a8895b3bf3b is 50, key is test_row_0/B:col10/1732436183742/Put/seqid=0 2024-11-24T08:16:24,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742387_1563 (size=12001) 2024-11-24T08:16:24,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436244909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:24,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436244920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:24,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436244920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:24,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:24,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/544632a2ea34479aa3376a8895b3bf3b 2024-11-24T08:16:24,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/0b79a2c8e5094cfa9c08d7f2b658a424 is 50, key is test_row_0/C:col10/1732436183742/Put/seqid=0 2024-11-24T08:16:24,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742388_1564 (size=12001) 2024-11-24T08:16:25,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/0b79a2c8e5094cfa9c08d7f2b658a424 2024-11-24T08:16:25,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/2d7ecd8406b54030bbc4f4842cda4e54 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/2d7ecd8406b54030bbc4f4842cda4e54 2024-11-24T08:16:25,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/2d7ecd8406b54030bbc4f4842cda4e54, entries=250, sequenceid=123, filesize=47.0 K 2024-11-24T08:16:25,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/544632a2ea34479aa3376a8895b3bf3b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/544632a2ea34479aa3376a8895b3bf3b 2024-11-24T08:16:25,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/544632a2ea34479aa3376a8895b3bf3b, entries=150, sequenceid=123, filesize=11.7 K 2024-11-24T08:16:25,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/0b79a2c8e5094cfa9c08d7f2b658a424 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/0b79a2c8e5094cfa9c08d7f2b658a424 2024-11-24T08:16:25,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/0b79a2c8e5094cfa9c08d7f2b658a424, entries=150, sequenceid=123, filesize=11.7 K 2024-11-24T08:16:25,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fe9967a2ec3ac6950518f327ac4152ca in 1656ms, sequenceid=123, compaction requested=true 2024-11-24T08:16:25,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:25,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:16:25,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:25,400 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:25,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:16:25,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:25,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:16:25,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:25,400 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:25,401 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:25,401 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110255 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:25,401 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/B is initiating minor compaction (all files) 2024-11-24T08:16:25,401 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/A is initiating minor compaction (all files) 2024-11-24T08:16:25,401 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/A in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:25,401 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/B in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:25,401 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/f255dea9a5184ac0b30df7b6733812fb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d0caff9623754bb48a85f72d025facac, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/544632a2ea34479aa3376a8895b3bf3b] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=35.4 K 2024-11-24T08:16:25,401 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/677df9f3fd46436eaa40986ccaa2b6c8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/cc64a699a1d74970a7ba2e7670e8838c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/2d7ecd8406b54030bbc4f4842cda4e54] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=107.7 K 2024-11-24T08:16:25,401 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:25,401 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/677df9f3fd46436eaa40986ccaa2b6c8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/cc64a699a1d74970a7ba2e7670e8838c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/2d7ecd8406b54030bbc4f4842cda4e54] 2024-11-24T08:16:25,401 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting f255dea9a5184ac0b30df7b6733812fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732436179142 2024-11-24T08:16:25,401 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 677df9f3fd46436eaa40986ccaa2b6c8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732436179142 2024-11-24T08:16:25,401 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting d0caff9623754bb48a85f72d025facac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732436180416 2024-11-24T08:16:25,402 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc64a699a1d74970a7ba2e7670e8838c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732436180416 2024-11-24T08:16:25,402 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 544632a2ea34479aa3376a8895b3bf3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732436182594 2024-11-24T08:16:25,402 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d7ecd8406b54030bbc4f4842cda4e54, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732436182594 2024-11-24T08:16:25,406 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:25,407 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#B#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:25,408 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/7257fd7faeef4a5db93a2b0d4d524fdf is 50, key is test_row_0/B:col10/1732436183742/Put/seqid=0 2024-11-24T08:16:25,409 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241124b64e1032b1fc454aa71ccf9417237db9_fe9967a2ec3ac6950518f327ac4152ca store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:25,411 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241124b64e1032b1fc454aa71ccf9417237db9_fe9967a2ec3ac6950518f327ac4152ca, store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:25,411 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124b64e1032b1fc454aa71ccf9417237db9_fe9967a2ec3ac6950518f327ac4152ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:25,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742389_1565 (size=12309) 2024-11-24T08:16:25,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742390_1566 (size=4469) 2024-11-24T08:16:25,431 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#A#compaction#478 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:25,432 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/4f8a7470242c41b0811c66c864bdcc80 is 175, key is test_row_0/A:col10/1732436183742/Put/seqid=0 2024-11-24T08:16:25,436 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/7257fd7faeef4a5db93a2b0d4d524fdf as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/7257fd7faeef4a5db93a2b0d4d524fdf 2024-11-24T08:16:25,441 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/B of fe9967a2ec3ac6950518f327ac4152ca into 7257fd7faeef4a5db93a2b0d4d524fdf(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:25,441 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:25,441 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/B, priority=13, startTime=1732436185400; duration=0sec 2024-11-24T08:16:25,441 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:25,441 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:B 2024-11-24T08:16:25,441 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:25,442 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:25,442 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/C is initiating minor compaction (all files) 2024-11-24T08:16:25,442 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/C in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:25,442 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/eb488fa369564abd99d14fd144d14b93, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/294b216ba71742ce96f606746f71fc2b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/0b79a2c8e5094cfa9c08d7f2b658a424] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=35.4 K 2024-11-24T08:16:25,443 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting eb488fa369564abd99d14fd144d14b93, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1732436179142 2024-11-24T08:16:25,443 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 294b216ba71742ce96f606746f71fc2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732436180416 2024-11-24T08:16:25,444 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b79a2c8e5094cfa9c08d7f2b658a424, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732436182594 2024-11-24T08:16:25,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742391_1567 (size=31263) 2024-11-24T08:16:25,452 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#C#compaction#479 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:25,452 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/fc2a01fab7ea4e1aad2cba998f836ef7 is 50, key is test_row_0/C:col10/1732436183742/Put/seqid=0 2024-11-24T08:16:25,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742392_1568 (size=12309) 2024-11-24T08:16:25,850 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/4f8a7470242c41b0811c66c864bdcc80 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/4f8a7470242c41b0811c66c864bdcc80 2024-11-24T08:16:25,854 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/A of fe9967a2ec3ac6950518f327ac4152ca into 4f8a7470242c41b0811c66c864bdcc80(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:25,855 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:25,855 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/A, priority=13, startTime=1732436185400; duration=0sec 2024-11-24T08:16:25,855 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:25,855 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:A 2024-11-24T08:16:25,859 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/fc2a01fab7ea4e1aad2cba998f836ef7 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fc2a01fab7ea4e1aad2cba998f836ef7 2024-11-24T08:16:25,870 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/C of fe9967a2ec3ac6950518f327ac4152ca into fc2a01fab7ea4e1aad2cba998f836ef7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:25,870 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:25,870 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/C, priority=13, startTime=1732436185400; duration=0sec 2024-11-24T08:16:25,870 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:25,870 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:C 2024-11-24T08:16:25,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:25,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T08:16:25,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:25,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:25,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:25,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:25,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:25,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:25,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bc745bba8d6942aeb1aaa3e3ad53c278_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436185933/Put/seqid=0 2024-11-24T08:16:25,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742393_1569 (size=12304) 2024-11-24T08:16:25,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436245956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:25,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:25,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436245959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:25,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:25,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436245961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:26,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436246069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:26,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436246070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:26,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436246070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:26,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436246279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436246279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436246279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,283 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:26,356 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:26,359 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bc745bba8d6942aeb1aaa3e3ad53c278_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bc745bba8d6942aeb1aaa3e3ad53c278_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:26,360 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/b29c7aaaee494118a46610fdfd6af82a, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:26,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/b29c7aaaee494118a46610fdfd6af82a is 175, key is test_row_0/A:col10/1732436185933/Put/seqid=0 2024-11-24T08:16:26,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742394_1570 (size=31105) 2024-11-24T08:16:26,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:26,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436246590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:26,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436246590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:26,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436246590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:26,592 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:26,766 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/b29c7aaaee494118a46610fdfd6af82a 2024-11-24T08:16:26,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/2780c3d5539c4c0cb49a31a5ca463f29 is 50, key is test_row_0/B:col10/1732436185933/Put/seqid=0 2024-11-24T08:16:26,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742395_1571 (size=12151) 2024-11-24T08:16:26,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/2780c3d5539c4c0cb49a31a5ca463f29 2024-11-24T08:16:26,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/1816c1b7ff724b4fadba4133063aff86 is 50, key is test_row_0/C:col10/1732436185933/Put/seqid=0 2024-11-24T08:16:26,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742396_1572 (size=12151) 2024-11-24T08:16:27,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436247100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:27,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:27,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436247100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:27,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:27,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:27,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436247100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:27,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/1816c1b7ff724b4fadba4133063aff86 2024-11-24T08:16:27,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/b29c7aaaee494118a46610fdfd6af82a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/b29c7aaaee494118a46610fdfd6af82a 2024-11-24T08:16:27,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/b29c7aaaee494118a46610fdfd6af82a, entries=150, sequenceid=144, filesize=30.4 K 2024-11-24T08:16:27,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/2780c3d5539c4c0cb49a31a5ca463f29 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/2780c3d5539c4c0cb49a31a5ca463f29 2024-11-24T08:16:27,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/2780c3d5539c4c0cb49a31a5ca463f29, entries=150, sequenceid=144, filesize=11.9 K 2024-11-24T08:16:27,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/1816c1b7ff724b4fadba4133063aff86 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/1816c1b7ff724b4fadba4133063aff86 2024-11-24T08:16:27,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/1816c1b7ff724b4fadba4133063aff86, entries=150, sequenceid=144, filesize=11.9 K 2024-11-24T08:16:27,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for fe9967a2ec3ac6950518f327ac4152ca in 1268ms, sequenceid=144, compaction requested=false 2024-11-24T08:16:27,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-24T08:16:27,240 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:27,240 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:16:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-24T08:16:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T08:16:27,242 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:16:27,242 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:16:27,242 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:16:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T08:16:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:27,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T08:16:27,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:27,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:27,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:27,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:27,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:27,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:27,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a92df305895e44839eac4c94ec6b18a5_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436185960/Put/seqid=0 2024-11-24T08:16:27,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742397_1573 (size=14794) 2024-11-24T08:16:27,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T08:16:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:27,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:27,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:27,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436247419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436247419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:27,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436247530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:27,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436247530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:27,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:27,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:27,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T08:16:27,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:27,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:27,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:27,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:27,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T08:16:27,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T08:16:27,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:27,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:27,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:27,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:27,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:27,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436247740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:27,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:27,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436247740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:27,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:27,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:27,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:27,763 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:27,766 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124a92df305895e44839eac4c94ec6b18a5_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a92df305895e44839eac4c94ec6b18a5_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:27,767 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/f6c025bcf09342d1b5b0c2f445eba90d, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:27,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/f6c025bcf09342d1b5b0c2f445eba90d is 175, key is test_row_0/A:col10/1732436185960/Put/seqid=0 2024-11-24T08:16:27,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742398_1574 (size=39749) 2024-11-24T08:16:27,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T08:16:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:27,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:27,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:27,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:27,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T08:16:28,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T08:16:28,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:28,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:28,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:28,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436248049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:28,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436248049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:28,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:28,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:28,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:28,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:28,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436248110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:28,114 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_0' in fe9967a2ec3ac6950518f327ac4152ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:28,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:28,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:28,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:28,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436248119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:28,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436248119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:28,123 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in fe9967a2ec3ac6950518f327ac4152ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:28,123 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in fe9967a2ec3ac6950518f327ac4152ca of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:28,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:28,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:28,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:28,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T08:16:28,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:28,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:28,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:28,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,172 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/f6c025bcf09342d1b5b0c2f445eba90d 2024-11-24T08:16:28,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/d17c24da11554ee4ab6a9f20f4cb1e41 is 50, key is test_row_0/B:col10/1732436185960/Put/seqid=0 2024-11-24T08:16:28,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742399_1575 (size=12151) 2024-11-24T08:16:28,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T08:16:28,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:28,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:28,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:28,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T08:16:28,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T08:16:28,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:28,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:28,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:28,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436248560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:28,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:28,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436248560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:28,563 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:28,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/d17c24da11554ee4ab6a9f20f4cb1e41 2024-11-24T08:16:28,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/fea31a60d02d49d8a5340e3e2b05c01e is 50, key is test_row_0/C:col10/1732436185960/Put/seqid=0 2024-11-24T08:16:28,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742400_1576 (size=12151) 2024-11-24T08:16:28,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T08:16:28,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:28,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:28,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T08:16:28,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:28,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:28,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:28,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T08:16:28,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:28,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:28,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:28,923 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:28,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/fea31a60d02d49d8a5340e3e2b05c01e 2024-11-24T08:16:29,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/f6c025bcf09342d1b5b0c2f445eba90d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f6c025bcf09342d1b5b0c2f445eba90d 2024-11-24T08:16:29,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f6c025bcf09342d1b5b0c2f445eba90d, entries=200, sequenceid=159, filesize=38.8 K 2024-11-24T08:16:29,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/d17c24da11554ee4ab6a9f20f4cb1e41 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d17c24da11554ee4ab6a9f20f4cb1e41 2024-11-24T08:16:29,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d17c24da11554ee4ab6a9f20f4cb1e41, entries=150, sequenceid=159, filesize=11.9 K 2024-11-24T08:16:29,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/fea31a60d02d49d8a5340e3e2b05c01e as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fea31a60d02d49d8a5340e3e2b05c01e 2024-11-24T08:16:29,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fea31a60d02d49d8a5340e3e2b05c01e, entries=150, sequenceid=159, filesize=11.9 K 2024-11-24T08:16:29,010 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for fe9967a2ec3ac6950518f327ac4152ca in 1658ms, sequenceid=159, compaction requested=true 2024-11-24T08:16:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:16:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:29,010 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:16:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:16:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:29,010 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:29,011 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:29,011 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/A is initiating minor compaction (all files) 2024-11-24T08:16:29,011 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/A in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:29,011 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/4f8a7470242c41b0811c66c864bdcc80, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/b29c7aaaee494118a46610fdfd6af82a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f6c025bcf09342d1b5b0c2f445eba90d] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=99.7 K 2024-11-24T08:16:29,011 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:29,012 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/4f8a7470242c41b0811c66c864bdcc80, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/b29c7aaaee494118a46610fdfd6af82a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f6c025bcf09342d1b5b0c2f445eba90d] 2024-11-24T08:16:29,012 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:29,012 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/B is initiating minor compaction (all files) 2024-11-24T08:16:29,012 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/B in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:29,012 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f8a7470242c41b0811c66c864bdcc80, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732436182594 2024-11-24T08:16:29,012 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/7257fd7faeef4a5db93a2b0d4d524fdf, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/2780c3d5539c4c0cb49a31a5ca463f29, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d17c24da11554ee4ab6a9f20f4cb1e41] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=35.8 K 2024-11-24T08:16:29,012 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting b29c7aaaee494118a46610fdfd6af82a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732436183765 2024-11-24T08:16:29,012 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 7257fd7faeef4a5db93a2b0d4d524fdf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732436182594 2024-11-24T08:16:29,013 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6c025bcf09342d1b5b0c2f445eba90d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732436185954 2024-11-24T08:16:29,013 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 2780c3d5539c4c0cb49a31a5ca463f29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732436183765 2024-11-24T08:16:29,013 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting d17c24da11554ee4ab6a9f20f4cb1e41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732436185957 2024-11-24T08:16:29,018 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:29,020 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#B#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:29,020 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/a06caa1348004990898e517a9541a80a is 50, key is test_row_0/B:col10/1732436185960/Put/seqid=0 2024-11-24T08:16:29,021 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411241e13592b660e48299b5c1ca7f2340a76_fe9967a2ec3ac6950518f327ac4152ca store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:29,022 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411241e13592b660e48299b5c1ca7f2340a76_fe9967a2ec3ac6950518f327ac4152ca, store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:29,022 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241e13592b660e48299b5c1ca7f2340a76_fe9967a2ec3ac6950518f327ac4152ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:29,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742401_1577 (size=12561) 2024-11-24T08:16:29,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742402_1578 (size=4469) 2024-11-24T08:16:29,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-24T08:16:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:29,076 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-24T08:16:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:29,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:29,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244f36432d48494a388080c73587dae81c_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436187399/Put/seqid=0 2024-11-24T08:16:29,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742403_1579 (size=12304) 2024-11-24T08:16:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T08:16:29,437 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/a06caa1348004990898e517a9541a80a as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/a06caa1348004990898e517a9541a80a 2024-11-24T08:16:29,441 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/B of fe9967a2ec3ac6950518f327ac4152ca into a06caa1348004990898e517a9541a80a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:29,441 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:29,441 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/B, priority=13, startTime=1732436189010; duration=0sec 2024-11-24T08:16:29,441 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:29,441 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:B 2024-11-24T08:16:29,441 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:29,442 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:29,442 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/C is initiating minor compaction (all files) 2024-11-24T08:16:29,442 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/C in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:29,442 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fc2a01fab7ea4e1aad2cba998f836ef7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/1816c1b7ff724b4fadba4133063aff86, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fea31a60d02d49d8a5340e3e2b05c01e] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=35.8 K 2024-11-24T08:16:29,443 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting fc2a01fab7ea4e1aad2cba998f836ef7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1732436182594 2024-11-24T08:16:29,443 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 1816c1b7ff724b4fadba4133063aff86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732436183765 2024-11-24T08:16:29,443 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#A#compaction#487 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:29,443 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting fea31a60d02d49d8a5340e3e2b05c01e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732436185957 2024-11-24T08:16:29,444 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/f786304a8c15494a9c883404c54971e8 is 175, key is test_row_0/A:col10/1732436185960/Put/seqid=0 2024-11-24T08:16:29,450 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#C#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:29,451 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/38c42ec87a124429bf9f0304119a8af2 is 50, key is test_row_0/C:col10/1732436185960/Put/seqid=0 2024-11-24T08:16:29,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742404_1580 (size=31515) 2024-11-24T08:16:29,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742405_1581 (size=12561) 2024-11-24T08:16:29,471 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/f786304a8c15494a9c883404c54971e8 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f786304a8c15494a9c883404c54971e8 2024-11-24T08:16:29,475 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/A of fe9967a2ec3ac6950518f327ac4152ca into f786304a8c15494a9c883404c54971e8(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:29,475 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:29,475 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/A, priority=13, startTime=1732436189010; duration=0sec 2024-11-24T08:16:29,475 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:29,475 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:A 2024-11-24T08:16:29,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:29,489 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411244f36432d48494a388080c73587dae81c_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244f36432d48494a388080c73587dae81c_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/78dce05dcc704c2b92fe610ed89822c3, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/78dce05dcc704c2b92fe610ed89822c3 is 175, key is test_row_0/A:col10/1732436187399/Put/seqid=0 2024-11-24T08:16:29,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742406_1582 (size=31105) 2024-11-24T08:16:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:29,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:29,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436249599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:29,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:29,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:29,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:29,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436249601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:29,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:29,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:29,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:29,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436249709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:29,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436249709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:29,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:29,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:29,870 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/38c42ec87a124429bf9f0304119a8af2 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/38c42ec87a124429bf9f0304119a8af2 2024-11-24T08:16:29,874 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/C of fe9967a2ec3ac6950518f327ac4152ca into 38c42ec87a124429bf9f0304119a8af2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:29,874 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:29,874 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/C, priority=13, startTime=1732436189010; duration=0sec 2024-11-24T08:16:29,874 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:29,874 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:C 2024-11-24T08:16:29,894 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/78dce05dcc704c2b92fe610ed89822c3 2024-11-24T08:16:29,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/4f61dcd03b6f445fb0f4320b1cac2051 is 50, key is test_row_0/B:col10/1732436187399/Put/seqid=0 2024-11-24T08:16:29,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742407_1583 (size=12151) 2024-11-24T08:16:29,910 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/4f61dcd03b6f445fb0f4320b1cac2051 2024-11-24T08:16:29,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/b65947519c7b4495b5201fa289d6cd8b is 50, key is test_row_0/C:col10/1732436187399/Put/seqid=0 2024-11-24T08:16:29,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742408_1584 (size=12151) 2024-11-24T08:16:29,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:29,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436249920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:29,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:29,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436249920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:29,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:30,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436250130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:30,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:30,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436250130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:30,132 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in fe9967a2ec3ac6950518f327ac4152ca of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:30,132 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_1' in fe9967a2ec3ac6950518f327ac4152ca of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:30,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:30,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436250140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:30,143 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 30c28c82771d,45087,1732436016722 for 'test_row_0' in fe9967a2ec3ac6950518f327ac4152ca of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-24T08:16:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:30,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436250230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:30,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436250230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:30,319 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/b65947519c7b4495b5201fa289d6cd8b 2024-11-24T08:16:30,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/78dce05dcc704c2b92fe610ed89822c3 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/78dce05dcc704c2b92fe610ed89822c3 2024-11-24T08:16:30,325 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/78dce05dcc704c2b92fe610ed89822c3, entries=150, sequenceid=177, filesize=30.4 K 2024-11-24T08:16:30,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/4f61dcd03b6f445fb0f4320b1cac2051 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/4f61dcd03b6f445fb0f4320b1cac2051 2024-11-24T08:16:30,329 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/4f61dcd03b6f445fb0f4320b1cac2051, entries=150, sequenceid=177, filesize=11.9 K 2024-11-24T08:16:30,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/b65947519c7b4495b5201fa289d6cd8b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/b65947519c7b4495b5201fa289d6cd8b 2024-11-24T08:16:30,333 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/b65947519c7b4495b5201fa289d6cd8b, entries=150, sequenceid=177, filesize=11.9 K 2024-11-24T08:16:30,334 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for fe9967a2ec3ac6950518f327ac4152ca in 1258ms, sequenceid=177, compaction requested=false 2024-11-24T08:16:30,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:30,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:30,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-24T08:16:30,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-24T08:16:30,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-24T08:16:30,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0940 sec 2024-11-24T08:16:30,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 3.0980 sec 2024-11-24T08:16:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:30,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-24T08:16:30,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:30,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:30,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:30,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:30,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:30,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:30,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f49c1d6f378f4e37a698d034d6da4560_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436190741/Put/seqid=0 2024-11-24T08:16:30,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742409_1585 (size=14794) 2024-11-24T08:16:30,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436250772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:30,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:30,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436250775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:30,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,779 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:30,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:30,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436250879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:30,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436250889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:31,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:31,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436251090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:31,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:31,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:31,095 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:31,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436251099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:31,103 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:31,155 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:31,158 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f49c1d6f378f4e37a698d034d6da4560_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f49c1d6f378f4e37a698d034d6da4560_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:31,159 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/41f8a34b8a22412690002b19620a6a9f, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:31,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/41f8a34b8a22412690002b19620a6a9f is 175, key is test_row_0/A:col10/1732436190741/Put/seqid=0 2024-11-24T08:16:31,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742410_1586 (size=39749) 2024-11-24T08:16:31,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436251399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:31,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:31,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:31,401 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-24T08:16:31,410 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:31,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:16:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-24T08:16:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T08:16:31,412 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:16:31,413 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:16:31,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:16:31,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:31,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436251410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:31,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:31,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:31,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:31,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T08:16:31,564 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/41f8a34b8a22412690002b19620a6a9f 2024-11-24T08:16:31,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T08:16:31,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:31,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:31,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:31,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:31,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:31,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/b9ec010e43174a0c870dfa88aa897055 is 50, key is test_row_0/B:col10/1732436190741/Put/seqid=0 2024-11-24T08:16:31,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742411_1587 (size=12151) 2024-11-24T08:16:31,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T08:16:31,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:31,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:31,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:31,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:31,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T08:16:31,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T08:16:31,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:31,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:31,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:31,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:31,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:31,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:31,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:31,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436251911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:31,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436251920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:31,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:31,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/b9ec010e43174a0c870dfa88aa897055 2024-11-24T08:16:31,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/ecc03a8c9bf54a74be0e7bd718dcb3f8 is 50, key is test_row_0/C:col10/1732436190741/Put/seqid=0 2024-11-24T08:16:31,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742412_1588 (size=12151) 2024-11-24T08:16:32,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T08:16:32,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:32,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:32,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:32,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:32,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:32,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T08:16:32,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T08:16:32,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:32,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:32,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:32,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:32,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:32,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:32,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T08:16:32,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:32,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:32,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:32,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:32,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:32,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:32,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/ecc03a8c9bf54a74be0e7bd718dcb3f8 2024-11-24T08:16:32,396 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/41f8a34b8a22412690002b19620a6a9f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/41f8a34b8a22412690002b19620a6a9f 2024-11-24T08:16:32,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/41f8a34b8a22412690002b19620a6a9f, entries=200, sequenceid=196, filesize=38.8 K 2024-11-24T08:16:32,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/b9ec010e43174a0c870dfa88aa897055 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/b9ec010e43174a0c870dfa88aa897055 2024-11-24T08:16:32,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/b9ec010e43174a0c870dfa88aa897055, entries=150, sequenceid=196, filesize=11.9 K 2024-11-24T08:16:32,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/ecc03a8c9bf54a74be0e7bd718dcb3f8 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/ecc03a8c9bf54a74be0e7bd718dcb3f8 2024-11-24T08:16:32,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/ecc03a8c9bf54a74be0e7bd718dcb3f8, entries=150, sequenceid=196, filesize=11.9 K 2024-11-24T08:16:32,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fe9967a2ec3ac6950518f327ac4152ca in 1674ms, sequenceid=196, compaction requested=true 2024-11-24T08:16:32,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:32,417 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:32,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:16:32,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:32,417 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:16:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:32,418 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:16:32,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:32,418 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/A is initiating minor compaction (all files) 2024-11-24T08:16:32,418 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:32,418 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/A in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:32,418 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/B is initiating minor compaction (all files) 2024-11-24T08:16:32,418 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f786304a8c15494a9c883404c54971e8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/78dce05dcc704c2b92fe610ed89822c3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/41f8a34b8a22412690002b19620a6a9f] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=100.0 K 2024-11-24T08:16:32,418 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/B in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:32,418 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:32,418 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f786304a8c15494a9c883404c54971e8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/78dce05dcc704c2b92fe610ed89822c3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/41f8a34b8a22412690002b19620a6a9f] 2024-11-24T08:16:32,418 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/a06caa1348004990898e517a9541a80a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/4f61dcd03b6f445fb0f4320b1cac2051, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/b9ec010e43174a0c870dfa88aa897055] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=36.0 K 2024-11-24T08:16:32,419 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting a06caa1348004990898e517a9541a80a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732436185957 2024-11-24T08:16:32,419 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f786304a8c15494a9c883404c54971e8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732436185957 2024-11-24T08:16:32,419 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f61dcd03b6f445fb0f4320b1cac2051, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732436187382 2024-11-24T08:16:32,419 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78dce05dcc704c2b92fe610ed89822c3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732436187382 2024-11-24T08:16:32,419 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting b9ec010e43174a0c870dfa88aa897055, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732436189597 2024-11-24T08:16:32,419 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41f8a34b8a22412690002b19620a6a9f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732436189593 2024-11-24T08:16:32,429 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:32,431 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:32,431 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/f315cc6965cc4083affa729fc17477c4 is 50, key is test_row_0/B:col10/1732436190741/Put/seqid=0 2024-11-24T08:16:32,436 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112476f0c38facd04a42b68eca188e933af6_fe9967a2ec3ac6950518f327ac4152ca store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:32,438 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112476f0c38facd04a42b68eca188e933af6_fe9967a2ec3ac6950518f327ac4152ca, store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:32,438 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112476f0c38facd04a42b68eca188e933af6_fe9967a2ec3ac6950518f327ac4152ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:32,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742413_1589 (size=12663) 2024-11-24T08:16:32,445 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/f315cc6965cc4083affa729fc17477c4 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/f315cc6965cc4083affa729fc17477c4 2024-11-24T08:16:32,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742414_1590 (size=4469) 2024-11-24T08:16:32,451 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/B of fe9967a2ec3ac6950518f327ac4152ca into f315cc6965cc4083affa729fc17477c4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:32,451 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:32,451 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/B, priority=13, startTime=1732436192417; duration=0sec 2024-11-24T08:16:32,451 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:32,451 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:B 2024-11-24T08:16:32,451 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:32,451 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#A#compaction#496 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:32,451 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/f00d9d023fe048f2b4af2e46d2222dff is 175, key is test_row_0/A:col10/1732436190741/Put/seqid=0 2024-11-24T08:16:32,453 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:32,453 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/C is initiating minor compaction (all files) 2024-11-24T08:16:32,453 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/C in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:32,453 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/38c42ec87a124429bf9f0304119a8af2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/b65947519c7b4495b5201fa289d6cd8b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/ecc03a8c9bf54a74be0e7bd718dcb3f8] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=36.0 K 2024-11-24T08:16:32,453 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 38c42ec87a124429bf9f0304119a8af2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732436185957 2024-11-24T08:16:32,454 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting b65947519c7b4495b5201fa289d6cd8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732436187382 2024-11-24T08:16:32,454 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting ecc03a8c9bf54a74be0e7bd718dcb3f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732436189597 2024-11-24T08:16:32,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742415_1591 (size=31617) 2024-11-24T08:16:32,465 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#C#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:32,465 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/fd4ca70eeab14ae793d5750c6a27a389 is 50, key is test_row_0/C:col10/1732436190741/Put/seqid=0 2024-11-24T08:16:32,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742416_1592 (size=12663) 2024-11-24T08:16:32,472 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/fd4ca70eeab14ae793d5750c6a27a389 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fd4ca70eeab14ae793d5750c6a27a389 2024-11-24T08:16:32,477 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/C of fe9967a2ec3ac6950518f327ac4152ca into fd4ca70eeab14ae793d5750c6a27a389(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:32,477 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:32,477 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/C, priority=13, startTime=1732436192418; duration=0sec 2024-11-24T08:16:32,477 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:32,477 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:C 2024-11-24T08:16:32,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-24T08:16:32,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:32,487 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T08:16:32,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:32,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:32,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:32,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:32,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:32,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:32,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bd81bfe58e6a438a916e07ea96834af6_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436190771/Put/seqid=0 2024-11-24T08:16:32,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742417_1593 (size=12304) 2024-11-24T08:16:32,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:32,514 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124bd81bfe58e6a438a916e07ea96834af6_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bd81bfe58e6a438a916e07ea96834af6_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:32,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/218de6a16b534797acf21e387b8a7f7b, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/218de6a16b534797acf21e387b8a7f7b is 175, key is test_row_0/A:col10/1732436190771/Put/seqid=0 2024-11-24T08:16:32,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742418_1594 (size=31105) 2024-11-24T08:16:32,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T08:16:32,866 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/f00d9d023fe048f2b4af2e46d2222dff as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f00d9d023fe048f2b4af2e46d2222dff 2024-11-24T08:16:32,871 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/A of fe9967a2ec3ac6950518f327ac4152ca into f00d9d023fe048f2b4af2e46d2222dff(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:32,871 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:32,871 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/A, priority=13, startTime=1732436192416; duration=0sec 2024-11-24T08:16:32,871 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:32,871 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:A 2024-11-24T08:16:32,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:32,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:32,932 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/218de6a16b534797acf21e387b8a7f7b 2024-11-24T08:16:32,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436252949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:32,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:32,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/8907ee12162746e58d0656f10cf9da10 is 50, key is test_row_0/B:col10/1732436190771/Put/seqid=0 2024-11-24T08:16:32,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436252953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:32,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742419_1595 (size=12151) 2024-11-24T08:16:32,962 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/8907ee12162746e58d0656f10cf9da10 2024-11-24T08:16:32,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/8c506d97c1e7439a879ab13dc9df5679 is 50, key is test_row_0/C:col10/1732436190771/Put/seqid=0 2024-11-24T08:16:32,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742420_1596 (size=12151) 2024-11-24T08:16:33,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:33,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436253060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:33,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:33,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436253060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,063 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:33,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436253270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:33,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:33,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436253270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:33,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:33,378 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/8c506d97c1e7439a879ab13dc9df5679 2024-11-24T08:16:33,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/218de6a16b534797acf21e387b8a7f7b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/218de6a16b534797acf21e387b8a7f7b 2024-11-24T08:16:33,386 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/218de6a16b534797acf21e387b8a7f7b, entries=150, sequenceid=214, filesize=30.4 K 2024-11-24T08:16:33,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/8907ee12162746e58d0656f10cf9da10 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/8907ee12162746e58d0656f10cf9da10 2024-11-24T08:16:33,403 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/8907ee12162746e58d0656f10cf9da10, entries=150, sequenceid=214, filesize=11.9 K 2024-11-24T08:16:33,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/8c506d97c1e7439a879ab13dc9df5679 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/8c506d97c1e7439a879ab13dc9df5679 2024-11-24T08:16:33,407 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/8c506d97c1e7439a879ab13dc9df5679, entries=150, sequenceid=214, filesize=11.9 K 2024-11-24T08:16:33,408 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fe9967a2ec3ac6950518f327ac4152ca in 921ms, sequenceid=214, compaction requested=false 2024-11-24T08:16:33,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:33,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:33,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-24T08:16:33,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-24T08:16:33,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-24T08:16:33,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9970 sec 2024-11-24T08:16:33,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.0030 sec 2024-11-24T08:16:33,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-24T08:16:33,561 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:33,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-24T08:16:33,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-24T08:16:33,564 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-24T08:16:33,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T08:16:33,565 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-24T08:16:33,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-24T08:16:33,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:33,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-24T08:16:33,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:33,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:33,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:33,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:33,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:33,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:33,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112415ba355989a240aaa74da4f6865ab129_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436192951/Put/seqid=0 2024-11-24T08:16:33,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742421_1597 (size=12304) 2024-11-24T08:16:33,603 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:33,607 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112415ba355989a240aaa74da4f6865ab129_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112415ba355989a240aaa74da4f6865ab129_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:33,609 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/2017d6a1a68e4141807508b3e4e92959, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:33,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/2017d6a1a68e4141807508b3e4e92959 is 175, key is test_row_0/A:col10/1732436192951/Put/seqid=0 2024-11-24T08:16:33,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:33,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436253617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:33,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:33,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436253629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:33,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,633 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:33,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742422_1598 (size=31105) 2024-11-24T08:16:33,639 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/2017d6a1a68e4141807508b3e4e92959 2024-11-24T08:16:33,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/3b2affdbe52442dcad2f25718266b757 is 50, key is test_row_0/B:col10/1732436192951/Put/seqid=0 2024-11-24T08:16:33,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742423_1599 (size=12151) 2024-11-24T08:16:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T08:16:33,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T08:16:33,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:33,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:33,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:33,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:33,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:33,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:33,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:33,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436253730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:33,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:33,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:33,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436253740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:33,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T08:16:33,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:33,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:33,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:33,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:33,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T08:16:33,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436253940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:33,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,942 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:33,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436253950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:33,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T08:16:34,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:34,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/3b2affdbe52442dcad2f25718266b757 2024-11-24T08:16:34,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/3083757c5d3f4769aa86ae7695a43d39 is 50, key is test_row_0/C:col10/1732436192951/Put/seqid=0 2024-11-24T08:16:34,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742424_1600 (size=12151) 2024-11-24T08:16:34,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/3083757c5d3f4769aa86ae7695a43d39 2024-11-24T08:16:34,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/2017d6a1a68e4141807508b3e4e92959 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/2017d6a1a68e4141807508b3e4e92959 2024-11-24T08:16:34,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/2017d6a1a68e4141807508b3e4e92959, entries=150, sequenceid=232, filesize=30.4 K 2024-11-24T08:16:34,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/3b2affdbe52442dcad2f25718266b757 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/3b2affdbe52442dcad2f25718266b757 2024-11-24T08:16:34,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/3b2affdbe52442dcad2f25718266b757, entries=150, sequenceid=232, filesize=11.9 K 2024-11-24T08:16:34,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/3083757c5d3f4769aa86ae7695a43d39 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/3083757c5d3f4769aa86ae7695a43d39 2024-11-24T08:16:34,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/3083757c5d3f4769aa86ae7695a43d39, entries=150, sequenceid=232, filesize=11.9 K 2024-11-24T08:16:34,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for fe9967a2ec3ac6950518f327ac4152ca in 547ms, sequenceid=232, compaction requested=true 2024-11-24T08:16:34,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:34,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-24T08:16:34,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:34,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-24T08:16:34,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:34,130 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:34,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fe9967a2ec3ac6950518f327ac4152ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-24T08:16:34,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:34,130 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:34,131 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:34,131 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:34,131 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/A is initiating minor compaction (all files) 2024-11-24T08:16:34,131 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/B is initiating minor compaction (all files) 2024-11-24T08:16:34,131 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/A in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,131 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/B in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,132 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f00d9d023fe048f2b4af2e46d2222dff, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/218de6a16b534797acf21e387b8a7f7b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/2017d6a1a68e4141807508b3e4e92959] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=91.6 K 2024-11-24T08:16:34,132 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/f315cc6965cc4083affa729fc17477c4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/8907ee12162746e58d0656f10cf9da10, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/3b2affdbe52442dcad2f25718266b757] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=36.1 K 2024-11-24T08:16:34,132 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,132 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. files: [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f00d9d023fe048f2b4af2e46d2222dff, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/218de6a16b534797acf21e387b8a7f7b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/2017d6a1a68e4141807508b3e4e92959] 2024-11-24T08:16:34,132 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting f315cc6965cc4083affa729fc17477c4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732436189597 2024-11-24T08:16:34,132 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting f00d9d023fe048f2b4af2e46d2222dff, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732436189597 2024-11-24T08:16:34,132 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 8907ee12162746e58d0656f10cf9da10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732436190767 2024-11-24T08:16:34,133 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 218de6a16b534797acf21e387b8a7f7b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732436190767 2024-11-24T08:16:34,133 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b2affdbe52442dcad2f25718266b757, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732436192945 2024-11-24T08:16:34,133 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2017d6a1a68e4141807508b3e4e92959, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732436192945 2024-11-24T08:16:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:34,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T08:16:34,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:34,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:34,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:34,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:34,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:34,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:34,146 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:34,150 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#B#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:34,151 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/a1192e4a4e4b470bb36265f5f418c5c6 is 50, key is test_row_0/B:col10/1732436192951/Put/seqid=0 2024-11-24T08:16:34,157 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411241a8180313970438baf57e0093ba8a4ed_fe9967a2ec3ac6950518f327ac4152ca store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:34,158 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411241a8180313970438baf57e0093ba8a4ed_fe9967a2ec3ac6950518f327ac4152ca, store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:34,158 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411241a8180313970438baf57e0093ba8a4ed_fe9967a2ec3ac6950518f327ac4152ca because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:34,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124212dcc5dd06f4f70bc27e18e3bc3a2b8_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436193616/Put/seqid=0 2024-11-24T08:16:34,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742425_1601 (size=12765) 2024-11-24T08:16:34,165 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/a1192e4a4e4b470bb36265f5f418c5c6 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/a1192e4a4e4b470bb36265f5f418c5c6 2024-11-24T08:16:34,170 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/B of fe9967a2ec3ac6950518f327ac4152ca into a1192e4a4e4b470bb36265f5f418c5c6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:34,170 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:34,170 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/B, priority=13, startTime=1732436194130; duration=0sec 2024-11-24T08:16:34,170 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-24T08:16:34,170 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:B 2024-11-24T08:16:34,170 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-24T08:16:34,173 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-24T08:16:34,173 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1541): fe9967a2ec3ac6950518f327ac4152ca/C is initiating minor compaction (all files) 2024-11-24T08:16:34,173 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fe9967a2ec3ac6950518f327ac4152ca/C in TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,173 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fd4ca70eeab14ae793d5750c6a27a389, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/8c506d97c1e7439a879ab13dc9df5679, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/3083757c5d3f4769aa86ae7695a43d39] into tmpdir=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp, totalSize=36.1 K 2024-11-24T08:16:34,174 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting fd4ca70eeab14ae793d5750c6a27a389, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732436189597 2024-11-24T08:16:34,174 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c506d97c1e7439a879ab13dc9df5679, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732436190767 2024-11-24T08:16:34,174 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] compactions.Compactor(225): Compacting 3083757c5d3f4769aa86ae7695a43d39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1732436192945 2024-11-24T08:16:34,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742427_1603 (size=14794) 2024-11-24T08:16:34,179 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:34,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T08:16:34,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:34,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,183 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#C#compaction#507 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:34,183 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124212dcc5dd06f4f70bc27e18e3bc3a2b8_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124212dcc5dd06f4f70bc27e18e3bc3a2b8_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:34,183 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/730cf71ebe874203ac564e6c8d247178 is 50, key is test_row_0/C:col10/1732436192951/Put/seqid=0 2024-11-24T08:16:34,184 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/b55168efe072405dae0776898055f6e3, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:34,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/b55168efe072405dae0776898055f6e3 is 175, key is test_row_0/A:col10/1732436193616/Put/seqid=0 2024-11-24T08:16:34,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742426_1602 (size=4469) 2024-11-24T08:16:34,188 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fe9967a2ec3ac6950518f327ac4152ca#A#compaction#504 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T08:16:34,188 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/b03c5802aac74f2f99d500a25c603057 is 175, key is test_row_0/A:col10/1732436192951/Put/seqid=0 2024-11-24T08:16:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T08:16:34,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742428_1604 (size=12765) 2024-11-24T08:16:34,197 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/730cf71ebe874203ac564e6c8d247178 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/730cf71ebe874203ac564e6c8d247178 2024-11-24T08:16:34,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436254193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436254194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,202 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/C of fe9967a2ec3ac6950518f327ac4152ca into 730cf71ebe874203ac564e6c8d247178(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:34,203 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:34,203 INFO [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/C, priority=13, startTime=1732436194130; duration=0sec 2024-11-24T08:16:34,203 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:34,203 DEBUG [RS:0;30c28c82771d:45087-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:C 2024-11-24T08:16:34,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436254199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,205 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742429_1605 (size=39749) 2024-11-24T08:16:34,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742430_1606 (size=31719) 2024-11-24T08:16:34,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436254250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436254260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,263 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436254310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436254310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436254310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T08:16:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T08:16:34,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:34,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436254520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436254520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436254521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,606 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=248, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/b55168efe072405dae0776898055f6e3 2024-11-24T08:16:34,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/322c9204a28f4bf3a999f015cf15d97f is 50, key is test_row_0/B:col10/1732436193616/Put/seqid=0 2024-11-24T08:16:34,615 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/b03c5802aac74f2f99d500a25c603057 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/b03c5802aac74f2f99d500a25c603057 2024-11-24T08:16:34,620 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fe9967a2ec3ac6950518f327ac4152ca/A of fe9967a2ec3ac6950518f327ac4152ca into b03c5802aac74f2f99d500a25c603057(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T08:16:34,620 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:34,620 INFO [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., storeName=fe9967a2ec3ac6950518f327ac4152ca/A, priority=13, startTime=1732436194130; duration=0sec 2024-11-24T08:16:34,621 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-24T08:16:34,621 DEBUG [RS:0;30c28c82771d:45087-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fe9967a2ec3ac6950518f327ac4152ca:A 2024-11-24T08:16:34,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742431_1607 (size=12151) 2024-11-24T08:16:34,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/322c9204a28f4bf3a999f015cf15d97f 2024-11-24T08:16:34,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/9581d8edbfff40daba82483ab0e64b97 is 50, key is test_row_0/C:col10/1732436193616/Put/seqid=0 2024-11-24T08:16:34,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T08:16:34,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:34,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742432_1608 (size=12151) 2024-11-24T08:16:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T08:16:34,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38352 deadline: 1732436254760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38376 deadline: 1732436254770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,774 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T08:16:34,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:34,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,798 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T08:16:34,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38324 deadline: 1732436254830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-24T08:16:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38310 deadline: 1732436254830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:38390 deadline: 1732436254830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 2024-11-24T08:16:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 , the old value is region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-24T08:16:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., hostname=30c28c82771d,45087,1732436016722, seqNum=5 because the exception is null or not the one we care about 2024-11-24T08:16:34,864 INFO [Thread-2373 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2373. 2024-11-24T08:16:34,864 INFO [Thread-2377 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2377. 2024-11-24T08:16:34,864 DEBUG [Thread-2373 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:34,864 DEBUG [Thread-2377 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:34,864 DEBUG [Thread-2377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:34,864 DEBUG [Thread-2373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:34,864 DEBUG [Thread-2377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:34,864 DEBUG [Thread-2373 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:34,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:34,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:34,864 INFO [Thread-2371 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2371. 2024-11-24T08:16:34,864 DEBUG [Thread-2371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:34,864 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:34,864 DEBUG [Thread-2371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:34,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:34,867 INFO [Thread-2375 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2375. 2024-11-24T08:16:34,867 DEBUG [Thread-2375 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:34,867 DEBUG [Thread-2375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:34,867 DEBUG [Thread-2375 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:34,867 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:34,868 INFO [Thread-2379 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2379. 2024-11-24T08:16:34,868 DEBUG [Thread-2379 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:34,868 DEBUG [Thread-2379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:34,868 DEBUG [Thread-2379 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:34,868 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:34,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T08:16:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:34,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T08:16:35,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/9581d8edbfff40daba82483ab0e64b97 2024-11-24T08:16:35,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/b55168efe072405dae0776898055f6e3 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/b55168efe072405dae0776898055f6e3 2024-11-24T08:16:35,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/b55168efe072405dae0776898055f6e3, entries=200, sequenceid=248, filesize=38.8 K 2024-11-24T08:16:35,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/322c9204a28f4bf3a999f015cf15d97f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/322c9204a28f4bf3a999f015cf15d97f 2024-11-24T08:16:35,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/322c9204a28f4bf3a999f015cf15d97f, entries=150, sequenceid=248, filesize=11.9 K 2024-11-24T08:16:35,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/9581d8edbfff40daba82483ab0e64b97 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/9581d8edbfff40daba82483ab0e64b97 2024-11-24T08:16:35,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/9581d8edbfff40daba82483ab0e64b97, entries=150, sequenceid=248, filesize=11.9 K 2024-11-24T08:16:35,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for fe9967a2ec3ac6950518f327ac4152ca in 938ms, sequenceid=248, compaction requested=false 2024-11-24T08:16:35,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:35,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45087 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-24T08:16:35,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:35,104 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-24T08:16:35,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:35,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:35,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:35,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:35,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:35,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:35,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e955ffcc8c294caa804e47bc3dc8f15d_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436194186/Put/seqid=0 2024-11-24T08:16:35,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742433_1609 (size=12454) 2024-11-24T08:16:35,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45087 {}] regionserver.HRegion(8855): Flush requested on fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:35,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. as already flushing 2024-11-24T08:16:35,341 INFO [Thread-2366 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2366. 2024-11-24T08:16:35,341 INFO [Thread-2364 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2364. 2024-11-24T08:16:35,341 DEBUG [Thread-2366 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:35,341 DEBUG [Thread-2364 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:35,341 DEBUG [Thread-2366 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:35,341 DEBUG [Thread-2364 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:35,341 DEBUG [Thread-2364 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:35,341 DEBUG [Thread-2366 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:35,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:35,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:35,351 INFO [Thread-2360 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2360. 2024-11-24T08:16:35,351 DEBUG [Thread-2360 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:35,351 DEBUG [Thread-2360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:35,351 DEBUG [Thread-2360 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:35,351 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:35,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:35,515 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124e955ffcc8c294caa804e47bc3dc8f15d_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e955ffcc8c294caa804e47bc3dc8f15d_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:35,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/18b7e596be874d538f558f804d915c31, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:35,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/18b7e596be874d538f558f804d915c31 is 175, key is test_row_0/A:col10/1732436194186/Put/seqid=0 2024-11-24T08:16:35,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742434_1610 (size=31255) 2024-11-24T08:16:35,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T08:16:35,771 INFO [Thread-2362 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2362. 2024-11-24T08:16:35,771 DEBUG [Thread-2362 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:35,771 DEBUG [Thread-2362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:35,771 DEBUG [Thread-2362 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:35,771 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:35,781 INFO [Thread-2368 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2368. 2024-11-24T08:16:35,781 DEBUG [Thread-2368 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-24T08:16:35,781 DEBUG [Thread-2368 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:35,781 DEBUG [Thread-2368 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:35,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:35,919 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/18b7e596be874d538f558f804d915c31 2024-11-24T08:16:35,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/6848ebaace1d4d58b0f9245fef31f6f1 is 50, key is test_row_0/B:col10/1732436194186/Put/seqid=0 2024-11-24T08:16:35,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742435_1611 (size=12301) 2024-11-24T08:16:36,328 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/6848ebaace1d4d58b0f9245fef31f6f1 2024-11-24T08:16:36,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/293abd4a5ea147958cf876ed751f3853 is 50, key is test_row_0/C:col10/1732436194186/Put/seqid=0 2024-11-24T08:16:36,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742436_1612 (size=12301) 2024-11-24T08:16:36,737 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/293abd4a5ea147958cf876ed751f3853 2024-11-24T08:16:36,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/18b7e596be874d538f558f804d915c31 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/18b7e596be874d538f558f804d915c31 2024-11-24T08:16:36,742 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/18b7e596be874d538f558f804d915c31, entries=150, sequenceid=267, filesize=30.5 K 2024-11-24T08:16:36,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/6848ebaace1d4d58b0f9245fef31f6f1 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/6848ebaace1d4d58b0f9245fef31f6f1 2024-11-24T08:16:36,745 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/6848ebaace1d4d58b0f9245fef31f6f1, entries=150, sequenceid=267, filesize=12.0 K 2024-11-24T08:16:36,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/293abd4a5ea147958cf876ed751f3853 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/293abd4a5ea147958cf876ed751f3853 2024-11-24T08:16:36,748 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/293abd4a5ea147958cf876ed751f3853, entries=150, sequenceid=267, filesize=12.0 K 2024-11-24T08:16:36,749 INFO [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=33.54 KB/34350 for fe9967a2ec3ac6950518f327ac4152ca in 1645ms, sequenceid=267, compaction requested=true 2024-11-24T08:16:36,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for fe9967a2ec3ac6950518f327ac4152ca: 2024-11-24T08:16:36,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:36,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/30c28c82771d:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-24T08:16:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-24T08:16:36,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-24T08:16:36,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1850 sec 2024-11-24T08:16:36,753 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 3.1900 sec 2024-11-24T08:16:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-24T08:16:37,730 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:37,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-24T08:16:37,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-24T08:16:37,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-24T08:16:37,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-24T08:16:37,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-24T08:16:37,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-24T08:16:37,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-24T08:16:37,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3018 2024-11-24T08:16:37,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3007 2024-11-24T08:16:37,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3042 2024-11-24T08:16:37,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3026 2024-11-24T08:16:37,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3016 2024-11-24T08:16:37,730 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-24T08:16:37,730 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T08:16:37,731 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:16:37,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:37,731 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:37,731 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:37,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-24T08:16:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T08:16:37,734 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436197734"}]},"ts":"1732436197734"} 2024-11-24T08:16:37,735 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-24T08:16:37,735 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-24T08:16:37,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-24T08:16:37,737 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, UNASSIGN}] 2024-11-24T08:16:37,737 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, UNASSIGN 2024-11-24T08:16:37,738 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=fe9967a2ec3ac6950518f327ac4152ca, regionState=CLOSING, regionLocation=30c28c82771d,45087,1732436016722 2024-11-24T08:16:37,738 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, UNASSIGN because future has completed 2024-11-24T08:16:37,739 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T08:16:37,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722}] 2024-11-24T08:16:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T08:16:37,891 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:37,891 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T08:16:37,891 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing fe9967a2ec3ac6950518f327ac4152ca, disabling compactions & flushes 2024-11-24T08:16:37,891 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:37,891 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:37,891 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. after waiting 0 ms 2024-11-24T08:16:37,891 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:37,891 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing fe9967a2ec3ac6950518f327ac4152ca 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-24T08:16:37,891 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=A 2024-11-24T08:16:37,891 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:37,891 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=B 2024-11-24T08:16:37,891 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:37,891 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK fe9967a2ec3ac6950518f327ac4152ca, store=C 2024-11-24T08:16:37,891 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-24T08:16:37,896 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f9cee451e2cc4fbc8bab97333a9bb6a0_fe9967a2ec3ac6950518f327ac4152ca is 50, key is test_row_0/A:col10/1732436195780/Put/seqid=0 2024-11-24T08:16:37,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742437_1613 (size=12454) 2024-11-24T08:16:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T08:16:38,299 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T08:16:38,302 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241124f9cee451e2cc4fbc8bab97333a9bb6a0_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f9cee451e2cc4fbc8bab97333a9bb6a0_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:38,302 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/aafcd7df32474c3baf9dceaf12f334b6, store: [table=TestAcidGuarantees family=A region=fe9967a2ec3ac6950518f327ac4152ca] 2024-11-24T08:16:38,303 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/aafcd7df32474c3baf9dceaf12f334b6 is 175, key is test_row_0/A:col10/1732436195780/Put/seqid=0 2024-11-24T08:16:38,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742438_1614 (size=31255) 2024-11-24T08:16:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T08:16:38,706 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=275, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/aafcd7df32474c3baf9dceaf12f334b6 2024-11-24T08:16:38,712 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/65512a22fecb46b9a6e76c34d2a945e3 is 50, key is test_row_0/B:col10/1732436195780/Put/seqid=0 2024-11-24T08:16:38,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742439_1615 (size=12301) 2024-11-24T08:16:38,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T08:16:39,115 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/65512a22fecb46b9a6e76c34d2a945e3 2024-11-24T08:16:39,121 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/b30b7d460f48443daa2d52e50f1c8c78 is 50, key is test_row_0/C:col10/1732436195780/Put/seqid=0 2024-11-24T08:16:39,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742440_1616 (size=12301) 2024-11-24T08:16:39,524 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/b30b7d460f48443daa2d52e50f1c8c78 2024-11-24T08:16:39,527 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/A/aafcd7df32474c3baf9dceaf12f334b6 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/aafcd7df32474c3baf9dceaf12f334b6 2024-11-24T08:16:39,530 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/aafcd7df32474c3baf9dceaf12f334b6, entries=150, sequenceid=275, filesize=30.5 K 2024-11-24T08:16:39,531 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/B/65512a22fecb46b9a6e76c34d2a945e3 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/65512a22fecb46b9a6e76c34d2a945e3 2024-11-24T08:16:39,533 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/65512a22fecb46b9a6e76c34d2a945e3, entries=150, sequenceid=275, filesize=12.0 K 2024-11-24T08:16:39,534 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/.tmp/C/b30b7d460f48443daa2d52e50f1c8c78 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/b30b7d460f48443daa2d52e50f1c8c78 2024-11-24T08:16:39,536 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/b30b7d460f48443daa2d52e50f1c8c78, entries=150, sequenceid=275, filesize=12.0 K 2024-11-24T08:16:39,537 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for fe9967a2ec3ac6950518f327ac4152ca in 1646ms, sequenceid=275, compaction requested=true 2024-11-24T08:16:39,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/eadd5c089f5d45c2a5269edfb7c2d7aa, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/8ed00444adc94bb4a376bdc272287be3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/1db5f06a5e05482a9f789ce0cd4d7ec0, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/3ee1264a839241dc8d68f72be7469b9f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/8755255240054b83a7f1fb74279e416b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/75f2d7bb8ec04c50bcb6a4d89ec88aee, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/677df9f3fd46436eaa40986ccaa2b6c8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/cc64a699a1d74970a7ba2e7670e8838c, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/2d7ecd8406b54030bbc4f4842cda4e54, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/4f8a7470242c41b0811c66c864bdcc80, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/b29c7aaaee494118a46610fdfd6af82a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f6c025bcf09342d1b5b0c2f445eba90d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f786304a8c15494a9c883404c54971e8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/78dce05dcc704c2b92fe610ed89822c3, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/41f8a34b8a22412690002b19620a6a9f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f00d9d023fe048f2b4af2e46d2222dff, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/218de6a16b534797acf21e387b8a7f7b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/2017d6a1a68e4141807508b3e4e92959] to archive 2024-11-24T08:16:39,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:16:39,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/eadd5c089f5d45c2a5269edfb7c2d7aa to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/eadd5c089f5d45c2a5269edfb7c2d7aa 2024-11-24T08:16:39,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/8ed00444adc94bb4a376bdc272287be3 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/8ed00444adc94bb4a376bdc272287be3 2024-11-24T08:16:39,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/1db5f06a5e05482a9f789ce0cd4d7ec0 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/1db5f06a5e05482a9f789ce0cd4d7ec0 2024-11-24T08:16:39,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/3ee1264a839241dc8d68f72be7469b9f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/3ee1264a839241dc8d68f72be7469b9f 2024-11-24T08:16:39,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/8755255240054b83a7f1fb74279e416b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/8755255240054b83a7f1fb74279e416b 2024-11-24T08:16:39,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/75f2d7bb8ec04c50bcb6a4d89ec88aee to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/75f2d7bb8ec04c50bcb6a4d89ec88aee 2024-11-24T08:16:39,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/677df9f3fd46436eaa40986ccaa2b6c8 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/677df9f3fd46436eaa40986ccaa2b6c8 2024-11-24T08:16:39,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/cc64a699a1d74970a7ba2e7670e8838c to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/cc64a699a1d74970a7ba2e7670e8838c 2024-11-24T08:16:39,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/2d7ecd8406b54030bbc4f4842cda4e54 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/2d7ecd8406b54030bbc4f4842cda4e54 2024-11-24T08:16:39,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/4f8a7470242c41b0811c66c864bdcc80 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/4f8a7470242c41b0811c66c864bdcc80 2024-11-24T08:16:39,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/b29c7aaaee494118a46610fdfd6af82a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/b29c7aaaee494118a46610fdfd6af82a 2024-11-24T08:16:39,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f6c025bcf09342d1b5b0c2f445eba90d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f6c025bcf09342d1b5b0c2f445eba90d 2024-11-24T08:16:39,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f786304a8c15494a9c883404c54971e8 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f786304a8c15494a9c883404c54971e8 2024-11-24T08:16:39,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/78dce05dcc704c2b92fe610ed89822c3 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/78dce05dcc704c2b92fe610ed89822c3 2024-11-24T08:16:39,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/41f8a34b8a22412690002b19620a6a9f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/41f8a34b8a22412690002b19620a6a9f 2024-11-24T08:16:39,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f00d9d023fe048f2b4af2e46d2222dff to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/f00d9d023fe048f2b4af2e46d2222dff 2024-11-24T08:16:39,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/218de6a16b534797acf21e387b8a7f7b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/218de6a16b534797acf21e387b8a7f7b 2024-11-24T08:16:39,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/2017d6a1a68e4141807508b3e4e92959 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/2017d6a1a68e4141807508b3e4e92959 2024-11-24T08:16:39,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/a93a3790727847f7a1a775af1e7aa1e9, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/c87cfd67e6f043a2bf72f5a56089f65f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/4cae532ca33f49b1bc0703737aabffa5, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d78a440fb4314423b56972080b89aa0f, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/08bba122ee7141e59420bb4a35ebb4dc, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/f255dea9a5184ac0b30df7b6733812fb, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/ee786f8951dc40388274eab8cdaf35af, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d0caff9623754bb48a85f72d025facac, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/7257fd7faeef4a5db93a2b0d4d524fdf, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/544632a2ea34479aa3376a8895b3bf3b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/2780c3d5539c4c0cb49a31a5ca463f29, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/a06caa1348004990898e517a9541a80a, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d17c24da11554ee4ab6a9f20f4cb1e41, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/4f61dcd03b6f445fb0f4320b1cac2051, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/f315cc6965cc4083affa729fc17477c4, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/b9ec010e43174a0c870dfa88aa897055, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/8907ee12162746e58d0656f10cf9da10, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/3b2affdbe52442dcad2f25718266b757] to archive 2024-11-24T08:16:39,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:16:39,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/a93a3790727847f7a1a775af1e7aa1e9 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/a93a3790727847f7a1a775af1e7aa1e9 2024-11-24T08:16:39,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/c87cfd67e6f043a2bf72f5a56089f65f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/c87cfd67e6f043a2bf72f5a56089f65f 2024-11-24T08:16:39,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/4cae532ca33f49b1bc0703737aabffa5 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/4cae532ca33f49b1bc0703737aabffa5 2024-11-24T08:16:39,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d78a440fb4314423b56972080b89aa0f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d78a440fb4314423b56972080b89aa0f 2024-11-24T08:16:39,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/08bba122ee7141e59420bb4a35ebb4dc to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/08bba122ee7141e59420bb4a35ebb4dc 2024-11-24T08:16:39,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/f255dea9a5184ac0b30df7b6733812fb to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/f255dea9a5184ac0b30df7b6733812fb 2024-11-24T08:16:39,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/ee786f8951dc40388274eab8cdaf35af to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/ee786f8951dc40388274eab8cdaf35af 2024-11-24T08:16:39,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d0caff9623754bb48a85f72d025facac to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d0caff9623754bb48a85f72d025facac 2024-11-24T08:16:39,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/7257fd7faeef4a5db93a2b0d4d524fdf to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/7257fd7faeef4a5db93a2b0d4d524fdf 2024-11-24T08:16:39,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/544632a2ea34479aa3376a8895b3bf3b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/544632a2ea34479aa3376a8895b3bf3b 2024-11-24T08:16:39,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/2780c3d5539c4c0cb49a31a5ca463f29 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/2780c3d5539c4c0cb49a31a5ca463f29 2024-11-24T08:16:39,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/a06caa1348004990898e517a9541a80a to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/a06caa1348004990898e517a9541a80a 2024-11-24T08:16:39,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d17c24da11554ee4ab6a9f20f4cb1e41 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/d17c24da11554ee4ab6a9f20f4cb1e41 2024-11-24T08:16:39,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/4f61dcd03b6f445fb0f4320b1cac2051 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/4f61dcd03b6f445fb0f4320b1cac2051 2024-11-24T08:16:39,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/f315cc6965cc4083affa729fc17477c4 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/f315cc6965cc4083affa729fc17477c4 2024-11-24T08:16:39,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/b9ec010e43174a0c870dfa88aa897055 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/b9ec010e43174a0c870dfa88aa897055 2024-11-24T08:16:39,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/8907ee12162746e58d0656f10cf9da10 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/8907ee12162746e58d0656f10cf9da10 2024-11-24T08:16:39,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/3b2affdbe52442dcad2f25718266b757 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/3b2affdbe52442dcad2f25718266b757 2024-11-24T08:16:39,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/081e8995d55c4ac58fc928528d5e65cf, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/e4e84fea69524192a346d31e834c5333, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/cb721af4b5824f9aa96ff5d17375139d, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/283b0ba367fa4973a20bec7170fb5910, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/233bddec61d940738a0724d3d30644da, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/eb488fa369564abd99d14fd144d14b93, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/d175b6b401a146c293484e9248bbd409, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/294b216ba71742ce96f606746f71fc2b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fc2a01fab7ea4e1aad2cba998f836ef7, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/0b79a2c8e5094cfa9c08d7f2b658a424, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/1816c1b7ff724b4fadba4133063aff86, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/38c42ec87a124429bf9f0304119a8af2, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fea31a60d02d49d8a5340e3e2b05c01e, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/b65947519c7b4495b5201fa289d6cd8b, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fd4ca70eeab14ae793d5750c6a27a389, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/ecc03a8c9bf54a74be0e7bd718dcb3f8, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/8c506d97c1e7439a879ab13dc9df5679, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/3083757c5d3f4769aa86ae7695a43d39] to archive 2024-11-24T08:16:39,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T08:16:39,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/081e8995d55c4ac58fc928528d5e65cf to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/081e8995d55c4ac58fc928528d5e65cf 2024-11-24T08:16:39,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/e4e84fea69524192a346d31e834c5333 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/e4e84fea69524192a346d31e834c5333 2024-11-24T08:16:39,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/cb721af4b5824f9aa96ff5d17375139d to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/cb721af4b5824f9aa96ff5d17375139d 2024-11-24T08:16:39,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/283b0ba367fa4973a20bec7170fb5910 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/283b0ba367fa4973a20bec7170fb5910 2024-11-24T08:16:39,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/233bddec61d940738a0724d3d30644da to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/233bddec61d940738a0724d3d30644da 2024-11-24T08:16:39,575 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/eb488fa369564abd99d14fd144d14b93 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/eb488fa369564abd99d14fd144d14b93 2024-11-24T08:16:39,576 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/d175b6b401a146c293484e9248bbd409 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/d175b6b401a146c293484e9248bbd409 2024-11-24T08:16:39,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/294b216ba71742ce96f606746f71fc2b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/294b216ba71742ce96f606746f71fc2b 2024-11-24T08:16:39,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fc2a01fab7ea4e1aad2cba998f836ef7 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fc2a01fab7ea4e1aad2cba998f836ef7 2024-11-24T08:16:39,578 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/0b79a2c8e5094cfa9c08d7f2b658a424 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/0b79a2c8e5094cfa9c08d7f2b658a424 2024-11-24T08:16:39,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/1816c1b7ff724b4fadba4133063aff86 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/1816c1b7ff724b4fadba4133063aff86 2024-11-24T08:16:39,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/38c42ec87a124429bf9f0304119a8af2 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/38c42ec87a124429bf9f0304119a8af2 2024-11-24T08:16:39,580 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fea31a60d02d49d8a5340e3e2b05c01e to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fea31a60d02d49d8a5340e3e2b05c01e 2024-11-24T08:16:39,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/b65947519c7b4495b5201fa289d6cd8b to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/b65947519c7b4495b5201fa289d6cd8b 2024-11-24T08:16:39,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fd4ca70eeab14ae793d5750c6a27a389 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/fd4ca70eeab14ae793d5750c6a27a389 2024-11-24T08:16:39,583 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/ecc03a8c9bf54a74be0e7bd718dcb3f8 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/ecc03a8c9bf54a74be0e7bd718dcb3f8 2024-11-24T08:16:39,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/8c506d97c1e7439a879ab13dc9df5679 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/8c506d97c1e7439a879ab13dc9df5679 2024-11-24T08:16:39,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/3083757c5d3f4769aa86ae7695a43d39 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/3083757c5d3f4769aa86ae7695a43d39 2024-11-24T08:16:39,588 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/recovered.edits/278.seqid, newMaxSeqId=278, maxSeqId=4 2024-11-24T08:16:39,589 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. 2024-11-24T08:16:39,589 DEBUG [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for fe9967a2ec3ac6950518f327ac4152ca: Waiting for close lock at 1732436197891Running coprocessor pre-close hooks at 1732436197891Disabling compacts and flushes for region at 1732436197891Disabling writes for close at 1732436197891Obtaining lock to block concurrent updates at 1732436197891Preparing flush snapshotting stores in fe9967a2ec3ac6950518f327ac4152ca at 1732436197891Finished memstore snapshotting TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732436197891Flushing stores of TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca. at 1732436197892 (+1 ms)Flushing fe9967a2ec3ac6950518f327ac4152ca/A: creating writer at 1732436197892Flushing mob file fe9967a2ec3ac6950518f327ac4152ca/A: appending metadata at 1732436197896 (+4 ms)Flushing mob file fe9967a2ec3ac6950518f327ac4152ca/A: closing flushed file at 1732436197896Flushing fe9967a2ec3ac6950518f327ac4152ca/A: appending metadata at 1732436198302 (+406 ms)Flushing fe9967a2ec3ac6950518f327ac4152ca/A: closing flushed file at 1732436198303 (+1 ms)Flushing fe9967a2ec3ac6950518f327ac4152ca/B: creating writer at 1732436198709 (+406 ms)Flushing fe9967a2ec3ac6950518f327ac4152ca/B: appending metadata at 1732436198711 (+2 ms)Flushing fe9967a2ec3ac6950518f327ac4152ca/B: closing flushed file at 1732436198711Flushing fe9967a2ec3ac6950518f327ac4152ca/C: creating writer at 1732436199118 (+407 ms)Flushing fe9967a2ec3ac6950518f327ac4152ca/C: appending metadata at 1732436199120 (+2 ms)Flushing fe9967a2ec3ac6950518f327ac4152ca/C: closing flushed file at 1732436199120Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@263c97bf: reopening flushed file at 1732436199527 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@236912df: reopening flushed file at 1732436199530 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cb0c18b: reopening flushed file at 1732436199533 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for fe9967a2ec3ac6950518f327ac4152ca in 1646ms, sequenceid=275, compaction requested=true at 1732436199537 (+4 ms)Writing region close event to WAL at 1732436199586 (+49 ms)Running coprocessor post-close hooks at 1732436199589 (+3 ms)Closed at 1732436199589 2024-11-24T08:16:39,590 INFO [RS_CLOSE_REGION-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,591 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=fe9967a2ec3ac6950518f327ac4152ca, regionState=CLOSED 2024-11-24T08:16:39,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 because future has completed 2024-11-24T08:16:39,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-24T08:16:39,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure fe9967a2ec3ac6950518f327ac4152ca, server=30c28c82771d,45087,1732436016722 in 1.8540 sec 2024-11-24T08:16:39,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-24T08:16:39,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=fe9967a2ec3ac6950518f327ac4152ca, UNASSIGN in 1.8570 sec 2024-11-24T08:16:39,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-24T08:16:39,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8610 sec 2024-11-24T08:16:39,598 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732436199598"}]},"ts":"1732436199598"} 2024-11-24T08:16:39,599 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-24T08:16:39,599 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-24T08:16:39,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8690 sec 2024-11-24T08:16:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-24T08:16:39,880 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:39,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-24T08:16:39,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:39,882 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T08:16:39,882 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:39,884 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,885 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C, FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/recovered.edits] 2024-11-24T08:16:39,887 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/18b7e596be874d538f558f804d915c31 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/18b7e596be874d538f558f804d915c31 2024-11-24T08:16:39,888 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/aafcd7df32474c3baf9dceaf12f334b6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/aafcd7df32474c3baf9dceaf12f334b6 2024-11-24T08:16:39,889 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/b03c5802aac74f2f99d500a25c603057 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/b03c5802aac74f2f99d500a25c603057 2024-11-24T08:16:39,890 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/b55168efe072405dae0776898055f6e3 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/A/b55168efe072405dae0776898055f6e3 2024-11-24T08:16:39,892 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/322c9204a28f4bf3a999f015cf15d97f to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/322c9204a28f4bf3a999f015cf15d97f 2024-11-24T08:16:39,893 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/65512a22fecb46b9a6e76c34d2a945e3 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/65512a22fecb46b9a6e76c34d2a945e3 2024-11-24T08:16:39,893 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/6848ebaace1d4d58b0f9245fef31f6f1 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/6848ebaace1d4d58b0f9245fef31f6f1 2024-11-24T08:16:39,894 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/a1192e4a4e4b470bb36265f5f418c5c6 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/B/a1192e4a4e4b470bb36265f5f418c5c6 2024-11-24T08:16:39,896 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/293abd4a5ea147958cf876ed751f3853 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/293abd4a5ea147958cf876ed751f3853 2024-11-24T08:16:39,897 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/730cf71ebe874203ac564e6c8d247178 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/730cf71ebe874203ac564e6c8d247178 2024-11-24T08:16:39,897 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/9581d8edbfff40daba82483ab0e64b97 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/9581d8edbfff40daba82483ab0e64b97 2024-11-24T08:16:39,898 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/b30b7d460f48443daa2d52e50f1c8c78 to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/C/b30b7d460f48443daa2d52e50f1c8c78 2024-11-24T08:16:39,900 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/recovered.edits/278.seqid to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca/recovered.edits/278.seqid 2024-11-24T08:16:39,901 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/default/TestAcidGuarantees/fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,901 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-24T08:16:39,902 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T08:16:39,902 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-24T08:16:39,904 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112411cc15afbb4f4103907cec38b9f823fa_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112411cc15afbb4f4103907cec38b9f823fa_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,905 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112415ba355989a240aaa74da4f6865ab129_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112415ba355989a240aaa74da4f6865ab129_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,906 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124212dcc5dd06f4f70bc27e18e3bc3a2b8_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124212dcc5dd06f4f70bc27e18e3bc3a2b8_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,906 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124263f96985bd44e188649ed90573e64ca_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124263f96985bd44e188649ed90573e64ca_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,907 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242991a133f32f45f6afae35b8b4d78184_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411242991a133f32f45f6afae35b8b4d78184_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,908 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244f36432d48494a388080c73587dae81c_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411244f36432d48494a388080c73587dae81c_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,909 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124713e32ec7e814de3a05fbce219a64f99_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124713e32ec7e814de3a05fbce219a64f99_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,910 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a92df305895e44839eac4c94ec6b18a5_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124a92df305895e44839eac4c94ec6b18a5_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,911 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bc745bba8d6942aeb1aaa3e3ad53c278_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bc745bba8d6942aeb1aaa3e3ad53c278_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,911 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bd81bfe58e6a438a916e07ea96834af6_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124bd81bfe58e6a438a916e07ea96834af6_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,912 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124be9fe413d07745788f3ad1d9f2deae71_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124be9fe413d07745788f3ad1d9f2deae71_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,913 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cebf9d8bc1a241c1b51edd9039d12b84_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124cebf9d8bc1a241c1b51edd9039d12b84_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,914 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d89fccc3ce6f4f5690ee538b6b59edc0_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124d89fccc3ce6f4f5690ee538b6b59edc0_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,914 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e955ffcc8c294caa804e47bc3dc8f15d_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124e955ffcc8c294caa804e47bc3dc8f15d_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,915 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f49c1d6f378f4e37a698d034d6da4560_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f49c1d6f378f4e37a698d034d6da4560_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,916 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f9cee451e2cc4fbc8bab97333a9bb6a0_fe9967a2ec3ac6950518f327ac4152ca to hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241124f9cee451e2cc4fbc8bab97333a9bb6a0_fe9967a2ec3ac6950518f327ac4152ca 2024-11-24T08:16:39,916 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-24T08:16:39,918 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:39,920 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-24T08:16:39,922 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-24T08:16:39,923 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:39,923 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-24T08:16:39,923 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732436199923"}]},"ts":"9223372036854775807"} 2024-11-24T08:16:39,925 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-24T08:16:39,925 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => fe9967a2ec3ac6950518f327ac4152ca, NAME => 'TestAcidGuarantees,,1732436172779.fe9967a2ec3ac6950518f327ac4152ca.', STARTKEY => '', ENDKEY => ''}] 2024-11-24T08:16:39,925 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-24T08:16:39,925 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732436199925"}]},"ts":"9223372036854775807"} 2024-11-24T08:16:39,926 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-24T08:16:39,926 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-24T08:16:39,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 46 msec 2024-11-24T08:16:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38469 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-24T08:16:39,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-24T08:16:39,990 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-24T08:16:39,999 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249 (was 250), OpenFileDescriptor=449 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=364 (was 340) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6169 (was 6241) 2024-11-24T08:16:39,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T08:16:39,999 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T08:16:39,999 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:16:39,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:39,999 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:39,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T08:16:40,000 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T08:16:40,000 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1570076000, stopped=false 2024-11-24T08:16:40,000 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=30c28c82771d,38469,1732436015766 2024-11-24T08:16:40,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T08:16:40,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:16:40,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T08:16:40,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:16:40,002 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T08:16:40,003 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:16:40,003 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T08:16:40,003 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:16:40,003 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:40,003 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T08:16:40,003 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '30c28c82771d,45087,1732436016722' ***** 2024-11-24T08:16:40,003 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T08:16:40,004 INFO [RS:0;30c28c82771d:45087 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T08:16:40,004 INFO [RS:0;30c28c82771d:45087 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T08:16:40,004 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T08:16:40,004 INFO [RS:0;30c28c82771d:45087 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T08:16:40,004 INFO [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(959): stopping server 30c28c82771d,45087,1732436016722 2024-11-24T08:16:40,004 INFO [RS:0;30c28c82771d:45087 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T08:16:40,004 INFO [RS:0;30c28c82771d:45087 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;30c28c82771d:45087. 2024-11-24T08:16:40,004 DEBUG [RS:0;30c28c82771d:45087 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T08:16:40,004 DEBUG [RS:0;30c28c82771d:45087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:40,005 INFO [RS:0;30c28c82771d:45087 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T08:16:40,005 INFO [RS:0;30c28c82771d:45087 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T08:16:40,005 INFO [RS:0;30c28c82771d:45087 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T08:16:40,005 INFO [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T08:16:40,005 INFO [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T08:16:40,006 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T08:16:40,006 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T08:16:40,006 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T08:16:40,006 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T08:16:40,006 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T08:16:40,006 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T08:16:40,006 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T08:16:40,006 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-24T08:16:40,029 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/.tmp/info/b4c05bd7ceb241ae8338b36ff2a8b66f is 95, key is TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831./info:/1732436054511/DeleteFamily/seqid=0 2024-11-24T08:16:40,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742441_1617 (size=6514) 2024-11-24T08:16:40,094 INFO [regionserver/30c28c82771d:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:16:40,206 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T08:16:40,406 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T08:16:40,433 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/.tmp/info/b4c05bd7ceb241ae8338b36ff2a8b66f 2024-11-24T08:16:40,451 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/.tmp/ns/99adac9eb00c4f0da3ad00de2778264d is 93, key is TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831./ns:/1732436054472/DeleteFamily/seqid=0 2024-11-24T08:16:40,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742442_1618 (size=5906) 2024-11-24T08:16:40,490 INFO [regionserver/30c28c82771d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T08:16:40,490 INFO [regionserver/30c28c82771d:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T08:16:40,606 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T08:16:40,807 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T08:16:40,854 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/.tmp/ns/99adac9eb00c4f0da3ad00de2778264d 2024-11-24T08:16:40,871 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/.tmp/rep_barrier/f88e06ed824c42e69bdbc2d0cd113527 is 102, key is TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831./rep_barrier:/1732436054472/DeleteFamily/seqid=0 2024-11-24T08:16:40,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742443_1619 (size=6025) 2024-11-24T08:16:41,007 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T08:16:41,207 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T08:16:41,275 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/.tmp/rep_barrier/f88e06ed824c42e69bdbc2d0cd113527 2024-11-24T08:16:41,297 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/.tmp/table/51637bef544940bbb5d672c454527f8b is 96, key is TestAcidGuarantees,,1732436019612.187d8f78b18a17144230e52cfd716831./table:/1732436054472/DeleteFamily/seqid=0 2024-11-24T08:16:41,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742444_1620 (size=5936) 2024-11-24T08:16:41,407 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T08:16:41,607 DEBUG [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T08:16:41,701 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/.tmp/table/51637bef544940bbb5d672c454527f8b 2024-11-24T08:16:41,704 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/.tmp/info/b4c05bd7ceb241ae8338b36ff2a8b66f as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/info/b4c05bd7ceb241ae8338b36ff2a8b66f 2024-11-24T08:16:41,707 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/info/b4c05bd7ceb241ae8338b36ff2a8b66f, entries=12, sequenceid=90, filesize=6.4 K 2024-11-24T08:16:41,707 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/.tmp/ns/99adac9eb00c4f0da3ad00de2778264d as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/ns/99adac9eb00c4f0da3ad00de2778264d 2024-11-24T08:16:41,710 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/ns/99adac9eb00c4f0da3ad00de2778264d, entries=8, sequenceid=90, filesize=5.8 K 2024-11-24T08:16:41,710 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/.tmp/rep_barrier/f88e06ed824c42e69bdbc2d0cd113527 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/rep_barrier/f88e06ed824c42e69bdbc2d0cd113527 2024-11-24T08:16:41,713 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/rep_barrier/f88e06ed824c42e69bdbc2d0cd113527, entries=6, sequenceid=90, filesize=5.9 K 2024-11-24T08:16:41,713 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/.tmp/table/51637bef544940bbb5d672c454527f8b as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/table/51637bef544940bbb5d672c454527f8b 2024-11-24T08:16:41,716 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/table/51637bef544940bbb5d672c454527f8b, entries=7, sequenceid=90, filesize=5.8 K 2024-11-24T08:16:41,716 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1710ms, sequenceid=90, compaction requested=false 2024-11-24T08:16:41,720 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-24T08:16:41,721 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T08:16:41,721 INFO [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T08:16:41,721 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732436200005Running coprocessor pre-close hooks at 1732436200006 (+1 ms)Disabling compacts and flushes for region at 1732436200006Disabling writes for close at 1732436200006Obtaining lock to block concurrent updates at 1732436200006Preparing flush snapshotting stores in 1588230740 at 1732436200006Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732436200007 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732436200007Flushing 1588230740/info: creating writer at 1732436200007Flushing 1588230740/info: appending metadata at 1732436200029 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732436200029Flushing 1588230740/ns: creating writer at 1732436200436 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732436200450 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732436200450Flushing 1588230740/rep_barrier: creating writer at 1732436200857 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732436200871 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732436200871Flushing 1588230740/table: creating writer at 1732436201278 (+407 ms)Flushing 1588230740/table: appending metadata at 1732436201297 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732436201297Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d28f030: reopening flushed file at 1732436201704 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@270d980a: reopening flushed file at 1732436201707 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ee78e83: reopening flushed file at 1732436201710 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@783b3303: reopening flushed file at 1732436201713 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1710ms, sequenceid=90, compaction requested=false at 1732436201716 (+3 ms)Writing region close event to WAL at 1732436201718 (+2 ms)Running coprocessor post-close hooks at 1732436201721 (+3 ms)Closed at 1732436201721 2024-11-24T08:16:41,721 DEBUG [RS_CLOSE_META-regionserver/30c28c82771d:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T08:16:41,808 INFO [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(976): stopping server 30c28c82771d,45087,1732436016722; all regions closed. 2024-11-24T08:16:41,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741834_1010 (size=25344) 2024-11-24T08:16:41,814 DEBUG [RS:0;30c28c82771d:45087 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/oldWALs 2024-11-24T08:16:41,814 INFO [RS:0;30c28c82771d:45087 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 30c28c82771d%2C45087%2C1732436016722.meta:.meta(num 1732436019226) 2024-11-24T08:16:41,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741833_1009 (size=17582910) 2024-11-24T08:16:41,817 DEBUG [RS:0;30c28c82771d:45087 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/oldWALs 2024-11-24T08:16:41,817 INFO [RS:0;30c28c82771d:45087 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 30c28c82771d%2C45087%2C1732436016722:(num 1732436018645) 2024-11-24T08:16:41,817 DEBUG [RS:0;30c28c82771d:45087 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T08:16:41,817 INFO [RS:0;30c28c82771d:45087 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T08:16:41,817 INFO [RS:0;30c28c82771d:45087 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T08:16:41,818 INFO [RS:0;30c28c82771d:45087 {}] hbase.ChoreService(370): Chore service for: regionserver/30c28c82771d:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T08:16:41,818 INFO [RS:0;30c28c82771d:45087 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T08:16:41,818 INFO [regionserver/30c28c82771d:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T08:16:41,819 INFO [RS:0;30c28c82771d:45087 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45087 2024-11-24T08:16:41,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/30c28c82771d,45087,1732436016722 2024-11-24T08:16:41,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T08:16:41,823 INFO [RS:0;30c28c82771d:45087 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T08:16:41,825 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [30c28c82771d,45087,1732436016722] 2024-11-24T08:16:41,827 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/30c28c82771d,45087,1732436016722 already deleted, retry=false 2024-11-24T08:16:41,827 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 30c28c82771d,45087,1732436016722 expired; onlineServers=0 2024-11-24T08:16:41,827 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '30c28c82771d,38469,1732436015766' ***** 2024-11-24T08:16:41,827 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T08:16:41,827 INFO [M:0;30c28c82771d:38469 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T08:16:41,827 INFO [M:0;30c28c82771d:38469 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T08:16:41,827 DEBUG [M:0;30c28c82771d:38469 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T08:16:41,827 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T08:16:41,827 DEBUG [M:0;30c28c82771d:38469 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T08:16:41,827 DEBUG [master/30c28c82771d:0:becomeActiveMaster-HFileCleaner.large.0-1732436018354 {}] cleaner.HFileCleaner(306): Exit Thread[master/30c28c82771d:0:becomeActiveMaster-HFileCleaner.large.0-1732436018354,5,FailOnTimeoutGroup] 2024-11-24T08:16:41,828 DEBUG [master/30c28c82771d:0:becomeActiveMaster-HFileCleaner.small.0-1732436018356 {}] cleaner.HFileCleaner(306): Exit Thread[master/30c28c82771d:0:becomeActiveMaster-HFileCleaner.small.0-1732436018356,5,FailOnTimeoutGroup] 2024-11-24T08:16:41,828 INFO [M:0;30c28c82771d:38469 {}] hbase.ChoreService(370): Chore service for: master/30c28c82771d:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T08:16:41,828 INFO [M:0;30c28c82771d:38469 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T08:16:41,828 DEBUG [M:0;30c28c82771d:38469 {}] master.HMaster(1795): Stopping service threads 2024-11-24T08:16:41,828 INFO [M:0;30c28c82771d:38469 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T08:16:41,828 INFO [M:0;30c28c82771d:38469 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T08:16:41,828 ERROR [M:0;30c28c82771d:38469 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-24T08:16:41,829 INFO [M:0;30c28c82771d:38469 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T08:16:41,829 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T08:16:41,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T08:16:41,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T08:16:41,831 DEBUG [M:0;30c28c82771d:38469 {}] zookeeper.ZKUtil(347): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T08:16:41,831 WARN [M:0;30c28c82771d:38469 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T08:16:41,831 INFO [M:0;30c28c82771d:38469 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/.lastflushedseqids 2024-11-24T08:16:41,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742445_1621 (size=72) 2024-11-24T08:16:41,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:16:41,926 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45087-0x1014909a19b0001, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:16:41,926 INFO [RS:0;30c28c82771d:45087 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T08:16:41,926 INFO [RS:0;30c28c82771d:45087 {}] regionserver.HRegionServer(1031): Exiting; stopping=30c28c82771d,45087,1732436016722; zookeeper connection closed. 2024-11-24T08:16:41,927 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7d207179 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7d207179 2024-11-24T08:16:41,927 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-24T08:16:42,241 INFO [M:0;30c28c82771d:38469 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T08:16:42,241 INFO [M:0;30c28c82771d:38469 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T08:16:42,241 DEBUG [M:0;30c28c82771d:38469 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T08:16:42,241 INFO [M:0;30c28c82771d:38469 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:16:42,241 DEBUG [M:0;30c28c82771d:38469 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:16:42,241 DEBUG [M:0;30c28c82771d:38469 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T08:16:42,241 DEBUG [M:0;30c28c82771d:38469 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:16:42,241 INFO [M:0;30c28c82771d:38469 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=777.95 KB heapSize=953.55 KB 2024-11-24T08:16:42,256 DEBUG [M:0;30c28c82771d:38469 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9587384239f04f9b94b2b53ab90fb618 is 82, key is hbase:meta,,1/info:regioninfo/1732436019315/Put/seqid=0 2024-11-24T08:16:42,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742446_1622 (size=5672) 2024-11-24T08:16:42,660 INFO [M:0;30c28c82771d:38469 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2172 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9587384239f04f9b94b2b53ab90fb618 2024-11-24T08:16:42,690 DEBUG [M:0;30c28c82771d:38469 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/43a5b57beef44639b26f37155fc57d86 is 2479, key is \x00\x00\x00\x00\x00\x00\x00`/proc:d/1732436120532/Put/seqid=0 2024-11-24T08:16:42,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742447_1623 (size=44995) 2024-11-24T08:16:43,094 INFO [M:0;30c28c82771d:38469 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=777.40 KB at sequenceid=2172 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/43a5b57beef44639b26f37155fc57d86 2024-11-24T08:16:43,097 INFO [M:0;30c28c82771d:38469 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 43a5b57beef44639b26f37155fc57d86 2024-11-24T08:16:43,112 DEBUG [M:0;30c28c82771d:38469 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6116d629c2fb44ccade7a9a9012ee6dc is 69, key is 30c28c82771d,45087,1732436016722/rs:state/1732436018401/Put/seqid=0 2024-11-24T08:16:43,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073742448_1624 (size=5156) 2024-11-24T08:16:43,515 INFO [M:0;30c28c82771d:38469 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2172 (bloomFilter=true), to=hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6116d629c2fb44ccade7a9a9012ee6dc 2024-11-24T08:16:43,519 DEBUG [M:0;30c28c82771d:38469 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9587384239f04f9b94b2b53ab90fb618 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9587384239f04f9b94b2b53ab90fb618 2024-11-24T08:16:43,522 INFO [M:0;30c28c82771d:38469 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9587384239f04f9b94b2b53ab90fb618, entries=8, sequenceid=2172, filesize=5.5 K 2024-11-24T08:16:43,522 DEBUG [M:0;30c28c82771d:38469 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/43a5b57beef44639b26f37155fc57d86 as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/43a5b57beef44639b26f37155fc57d86 2024-11-24T08:16:43,525 INFO [M:0;30c28c82771d:38469 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 43a5b57beef44639b26f37155fc57d86 2024-11-24T08:16:43,525 INFO [M:0;30c28c82771d:38469 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/43a5b57beef44639b26f37155fc57d86, entries=168, sequenceid=2172, filesize=43.9 K 2024-11-24T08:16:43,525 DEBUG [M:0;30c28c82771d:38469 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6116d629c2fb44ccade7a9a9012ee6dc as hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6116d629c2fb44ccade7a9a9012ee6dc 2024-11-24T08:16:43,527 INFO [M:0;30c28c82771d:38469 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44153/user/jenkins/test-data/0770d958-c0b7-46bc-b8ec-90e871b74af6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6116d629c2fb44ccade7a9a9012ee6dc, entries=1, sequenceid=2172, filesize=5.0 K 2024-11-24T08:16:43,528 INFO [M:0;30c28c82771d:38469 {}] regionserver.HRegion(3140): Finished flush of dataSize ~777.95 KB/796623, heapSize ~953.25 KB/976128, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1287ms, sequenceid=2172, compaction requested=false 2024-11-24T08:16:43,529 INFO [M:0;30c28c82771d:38469 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T08:16:43,529 DEBUG [M:0;30c28c82771d:38469 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732436202241Disabling compacts and flushes for region at 1732436202241Disabling writes for close at 1732436202241Obtaining lock to block concurrent updates at 1732436202241Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732436202241Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=796623, getHeapSize=976368, getOffHeapSize=0, getCellsCount=2365 at 1732436202241Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732436202242 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732436202242Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732436202256 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732436202256Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732436202664 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732436202690 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732436202690Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732436203097 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732436203111 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732436203111Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51f285c2: reopening flushed file at 1732436203519 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b77abae: reopening flushed file at 1732436203522 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a8254f: reopening flushed file at 1732436203525 (+3 ms)Finished flush of dataSize ~777.95 KB/796623, heapSize ~953.25 KB/976128, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1287ms, sequenceid=2172, compaction requested=false at 1732436203528 (+3 ms)Writing region close event to WAL at 1732436203529 (+1 ms)Closed at 1732436203529 2024-11-24T08:16:43,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46185 is added to blk_1073741830_1006 (size=938994) 2024-11-24T08:16:43,532 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T08:16:43,532 INFO [M:0;30c28c82771d:38469 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T08:16:43,533 INFO [M:0;30c28c82771d:38469 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38469 2024-11-24T08:16:43,533 INFO [M:0;30c28c82771d:38469 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T08:16:43,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:16:43,635 INFO [M:0;30c28c82771d:38469 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T08:16:43,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38469-0x1014909a19b0000, quorum=127.0.0.1:63500, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T08:16:43,639 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T08:16:43,641 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T08:16:43,642 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T08:16:43,642 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T08:16:43,642 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/hadoop.log.dir/,STOPPED} 2024-11-24T08:16:43,644 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T08:16:43,644 WARN [BP-1872930679-172.17.0.2-1732436012737 heartbeating to localhost/127.0.0.1:44153 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T08:16:43,644 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T08:16:43,644 WARN [BP-1872930679-172.17.0.2-1732436012737 heartbeating to localhost/127.0.0.1:44153 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1872930679-172.17.0.2-1732436012737 (Datanode Uuid 67fa83ec-f3c9-41c1-adf0-2436a7ad0421) service to localhost/127.0.0.1:44153 2024-11-24T08:16:43,646 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/cluster_6ca8f465-b694-0817-15a9-f097eff6822b/data/data1/current/BP-1872930679-172.17.0.2-1732436012737 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T08:16:43,647 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/cluster_6ca8f465-b694-0817-15a9-f097eff6822b/data/data2/current/BP-1872930679-172.17.0.2-1732436012737 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T08:16:43,647 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T08:16:43,657 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T08:16:43,658 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T08:16:43,658 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T08:16:43,658 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T08:16:43,658 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/217695b3-4faf-1384-95a5-588c87cd5369/hadoop.log.dir/,STOPPED} 2024-11-24T08:16:43,674 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T08:16:43,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down